2024-11-08 02:22:22,039 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-08 02:22:22,053 main DEBUG Took 0.012184 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 02:22:22,054 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 02:22:22,054 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 02:22:22,055 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 02:22:22,057 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,065 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 02:22:22,077 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,079 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,080 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,081 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,082 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,083 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,084 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,085 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,086 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,087 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,087 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,088 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:22:22,089 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,089 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 02:22:22,091 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:22:22,093 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 02:22:22,095 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 02:22:22,096 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 02:22:22,097 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 02:22:22,098 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 02:22:22,106 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 02:22:22,109 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 02:22:22,111 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 02:22:22,112 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 02:22:22,112 main DEBUG createAppenders(={Console}) 2024-11-08 02:22:22,113 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-08 02:22:22,113 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-08 02:22:22,113 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-08 02:22:22,114 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 02:22:22,114 main DEBUG OutputStream closed 2024-11-08 02:22:22,114 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 02:22:22,115 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 02:22:22,115 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-08 02:22:22,203 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 02:22:22,206 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 02:22:22,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 02:22:22,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 02:22:22,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 02:22:22,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 02:22:22,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 02:22:22,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 02:22:22,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 02:22:22,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 02:22:22,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 02:22:22,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 02:22:22,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 02:22:22,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 02:22:22,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 02:22:22,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 02:22:22,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 02:22:22,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 02:22:22,221 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 02:22:22,222 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-08 02:22:22,222 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 02:22:22,223 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-08T02:22:22,542 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4 2024-11-08 02:22:22,547 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 02:22:22,548 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T02:22:22,562 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-08T02:22:22,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T02:22:22,639 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/cluster_2d5389f6-e880-a508-0df3-b950eb02e0c6, deleteOnExit=true 2024-11-08T02:22:22,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-08T02:22:22,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/test.cache.data in system properties and HBase conf 2024-11-08T02:22:22,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T02:22:22,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/hadoop.log.dir in system properties and HBase conf 2024-11-08T02:22:22,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T02:22:22,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T02:22:22,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T02:22:22,776 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T02:22:22,917 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T02:22:22,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T02:22:22,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T02:22:22,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T02:22:22,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T02:22:22,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T02:22:22,927 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T02:22:22,928 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T02:22:22,929 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T02:22:22,929 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T02:22:22,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/nfs.dump.dir in system properties and HBase conf 2024-11-08T02:22:22,931 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/java.io.tmpdir in system properties and HBase conf 2024-11-08T02:22:22,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T02:22:22,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T02:22:22,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T02:22:24,136 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T02:22:24,216 INFO [Time-limited test {}] log.Log(170): Logging initialized @3144ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T02:22:24,293 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T02:22:24,368 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T02:22:24,394 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T02:22:24,394 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T02:22:24,395 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T02:22:24,411 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T02:22:24,416 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/hadoop.log.dir/,AVAILABLE} 2024-11-08T02:22:24,417 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T02:22:24,664 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/java.io.tmpdir/jetty-localhost-38717-hadoop-hdfs-3_4_1-tests_jar-_-any-1561027405544778629/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T02:22:24,673 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:38717} 2024-11-08T02:22:24,674 INFO [Time-limited test {}] server.Server(415): Started @3602ms 2024-11-08T02:22:25,271 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T02:22:25,280 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T02:22:25,282 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T02:22:25,282 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T02:22:25,282 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T02:22:25,283 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/hadoop.log.dir/,AVAILABLE} 2024-11-08T02:22:25,284 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T02:22:25,402 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/java.io.tmpdir/jetty-localhost-42049-hadoop-hdfs-3_4_1-tests_jar-_-any-6333155814215694720/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T02:22:25,402 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:42049} 2024-11-08T02:22:25,403 INFO [Time-limited test {}] server.Server(415): Started @4331ms 2024-11-08T02:22:25,456 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T02:22:26,457 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/cluster_2d5389f6-e880-a508-0df3-b950eb02e0c6/data/data2/current/BP-952722920-172.17.0.2-1731032543671/current, will proceed with Du for space computation calculation, 2024-11-08T02:22:26,457 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/cluster_2d5389f6-e880-a508-0df3-b950eb02e0c6/data/data1/current/BP-952722920-172.17.0.2-1731032543671/current, will proceed with Du for space computation calculation, 2024-11-08T02:22:26,492 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T02:22:26,536 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb9f106a14faf9ace with lease ID 0x78f03367e8075bf2: Processing first storage report for DS-3afd9e80-6193-4ae9-ba38-1bba3b9e56eb from datanode DatanodeRegistration(127.0.0.1:40173, datanodeUuid=c25ea9b5-d729-450a-a489-7c0aead6d2a0, infoPort=34541, infoSecurePort=0, ipcPort=33531, storageInfo=lv=-57;cid=testClusterID;nsid=845021141;c=1731032543671) 2024-11-08T02:22:26,538 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb9f106a14faf9ace with lease ID 0x78f03367e8075bf2: from storage DS-3afd9e80-6193-4ae9-ba38-1bba3b9e56eb node DatanodeRegistration(127.0.0.1:40173, datanodeUuid=c25ea9b5-d729-450a-a489-7c0aead6d2a0, infoPort=34541, infoSecurePort=0, ipcPort=33531, storageInfo=lv=-57;cid=testClusterID;nsid=845021141;c=1731032543671), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-08T02:22:26,539 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb9f106a14faf9ace with lease ID 0x78f03367e8075bf2: Processing first storage report for DS-4feddf07-420e-4d18-8a19-f3cae513f08a from datanode DatanodeRegistration(127.0.0.1:40173, datanodeUuid=c25ea9b5-d729-450a-a489-7c0aead6d2a0, infoPort=34541, infoSecurePort=0, ipcPort=33531, storageInfo=lv=-57;cid=testClusterID;nsid=845021141;c=1731032543671) 2024-11-08T02:22:26,539 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb9f106a14faf9ace with lease ID 0x78f03367e8075bf2: from storage DS-4feddf07-420e-4d18-8a19-f3cae513f08a node DatanodeRegistration(127.0.0.1:40173, datanodeUuid=c25ea9b5-d729-450a-a489-7c0aead6d2a0, infoPort=34541, infoSecurePort=0, ipcPort=33531, storageInfo=lv=-57;cid=testClusterID;nsid=845021141;c=1731032543671), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T02:22:26,573 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4 2024-11-08T02:22:26,636 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/cluster_2d5389f6-e880-a508-0df3-b950eb02e0c6/zookeeper_0, clientPort=52970, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/cluster_2d5389f6-e880-a508-0df3-b950eb02e0c6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/cluster_2d5389f6-e880-a508-0df3-b950eb02e0c6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T02:22:26,645 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52970 2024-11-08T02:22:26,658 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:22:26,661 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:22:26,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741825_1001 (size=7) 2024-11-08T02:22:27,269 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee with version=8 2024-11-08T02:22:27,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/hbase-staging 2024-11-08T02:22:27,351 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T02:22:27,570 INFO [Time-limited test {}] client.ConnectionUtils(128): master/331c7316141f:0 server-side Connection retries=45 2024-11-08T02:22:27,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T02:22:27,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T02:22:27,586 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T02:22:27,586 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T02:22:27,586 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T02:22:27,723 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T02:22:27,775 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T02:22:27,786 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T02:22:27,790 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T02:22:27,819 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 30827 (auto-detected) 2024-11-08T02:22:27,820 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T02:22:27,837 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41605 2024-11-08T02:22:27,857 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41605 connecting to ZooKeeper ensemble=127.0.0.1:52970 2024-11-08T02:22:27,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:416050x0, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T02:22:27,950 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41605-0x101184078e70000 connected 2024-11-08T02:22:28,039 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:22:28,043 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:22:28,056 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T02:22:28,059 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee, hbase.cluster.distributed=false 2024-11-08T02:22:28,081 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T02:22:28,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41605 2024-11-08T02:22:28,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41605 2024-11-08T02:22:28,087 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41605 2024-11-08T02:22:28,087 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41605 2024-11-08T02:22:28,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41605 2024-11-08T02:22:28,179 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/331c7316141f:0 server-side Connection retries=45 2024-11-08T02:22:28,180 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T02:22:28,180 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T02:22:28,181 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T02:22:28,181 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T02:22:28,181 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T02:22:28,183 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T02:22:28,186 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T02:22:28,187 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34325 2024-11-08T02:22:28,189 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34325 connecting to ZooKeeper ensemble=127.0.0.1:52970 2024-11-08T02:22:28,190 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:22:28,193 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:22:28,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:343250x0, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T02:22:28,205 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:343250x0, quorum=127.0.0.1:52970, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T02:22:28,205 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34325-0x101184078e70001 connected 2024-11-08T02:22:28,209 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T02:22:28,215 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T02:22:28,218 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T02:22:28,223 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T02:22:28,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34325 2024-11-08T02:22:28,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34325 2024-11-08T02:22:28,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34325 2024-11-08T02:22:28,226 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34325 2024-11-08T02:22:28,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34325 2024-11-08T02:22:28,240 DEBUG [M:0;331c7316141f:41605 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;331c7316141f:41605 2024-11-08T02:22:28,241 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/331c7316141f,41605,1731032547397 2024-11-08T02:22:28,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T02:22:28,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T02:22:28,259 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/331c7316141f,41605,1731032547397 2024-11-08T02:22:28,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:28,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T02:22:28,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:28,289 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T02:22:28,290 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/331c7316141f,41605,1731032547397 from backup master directory 2024-11-08T02:22:28,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/331c7316141f,41605,1731032547397 2024-11-08T02:22:28,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T02:22:28,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T02:22:28,300 WARN [master/331c7316141f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T02:22:28,300 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=331c7316141f,41605,1731032547397 2024-11-08T02:22:28,302 INFO [master/331c7316141f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T02:22:28,303 INFO [master/331c7316141f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T02:22:28,358 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/hbase.id] with ID: ad3e781f-9d7b-4875-a9cb-20151d78056f 2024-11-08T02:22:28,359 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/.tmp/hbase.id 2024-11-08T02:22:28,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741826_1002 (size=42) 2024-11-08T02:22:28,770 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/.tmp/hbase.id]:[hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/hbase.id] 2024-11-08T02:22:28,818 INFO [master/331c7316141f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:22:28,823 INFO [master/331c7316141f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T02:22:28,841 INFO [master/331c7316141f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-08T02:22:28,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:28,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:28,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741827_1003 (size=196) 2024-11-08T02:22:29,285 INFO [master/331c7316141f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:22:29,287 INFO [master/331c7316141f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T02:22:29,301 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:29,305 INFO [master/331c7316141f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T02:22:29,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741828_1004 (size=1189) 2024-11-08T02:22:29,756 INFO [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store 2024-11-08T02:22:29,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741829_1005 (size=34) 2024-11-08T02:22:30,183 INFO [master/331c7316141f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T02:22:30,188 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:22:30,189 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T02:22:30,190 INFO [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:22:30,190 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:22:30,192 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T02:22:30,192 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:22:30,192 INFO [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:22:30,194 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731032550189Disabling compacts and flushes for region at 1731032550189Disabling writes for close at 1731032550192 (+3 ms)Writing region close event to WAL at 1731032550192Closed at 1731032550192 2024-11-08T02:22:30,197 WARN [master/331c7316141f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/.initializing 2024-11-08T02:22:30,197 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/WALs/331c7316141f,41605,1731032547397 2024-11-08T02:22:30,207 INFO [master/331c7316141f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T02:22:30,226 INFO [master/331c7316141f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=331c7316141f%2C41605%2C1731032547397, suffix=, logDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/WALs/331c7316141f,41605,1731032547397, archiveDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/oldWALs, maxLogs=10 2024-11-08T02:22:30,256 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/WALs/331c7316141f,41605,1731032547397/331c7316141f%2C41605%2C1731032547397.1731032550231, exclude list is [], retry=0 2024-11-08T02:22:30,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40173,DS-3afd9e80-6193-4ae9-ba38-1bba3b9e56eb,DISK] 2024-11-08T02:22:30,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T02:22:30,337 INFO [master/331c7316141f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/WALs/331c7316141f,41605,1731032547397/331c7316141f%2C41605%2C1731032547397.1731032550231 2024-11-08T02:22:30,339 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34541:34541)] 2024-11-08T02:22:30,339 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:22:30,340 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:22:30,345 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,347 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,399 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T02:22:30,441 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:30,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:22:30,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T02:22:30,452 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:30,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:22:30,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T02:22:30,460 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:30,461 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:22:30,462 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,466 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T02:22:30,467 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:30,469 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:22:30,469 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,476 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,478 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,486 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,486 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,491 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T02:22:30,496 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:22:30,503 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:22:30,505 INFO [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74977522, jitterRate=0.11725214123725891}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T02:22:30,511 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731032550364Initializing all the Stores at 1731032550367 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731032550368 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032550369 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032550369Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032550369Cleaning up temporary data from old regions at 1731032550487 (+118 ms)Region opened successfully at 1731032550511 (+24 ms) 2024-11-08T02:22:30,513 INFO [master/331c7316141f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T02:22:30,551 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e5d436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=331c7316141f/172.17.0.2:0 2024-11-08T02:22:30,586 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T02:22:30,601 INFO [master/331c7316141f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T02:22:30,601 INFO [master/331c7316141f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T02:22:30,605 INFO [master/331c7316141f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T02:22:30,607 INFO [master/331c7316141f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T02:22:30,613 INFO [master/331c7316141f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-08T02:22:30,613 INFO [master/331c7316141f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T02:22:30,643 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T02:22:30,656 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T02:22:30,720 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T02:22:30,722 INFO [master/331c7316141f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T02:22:30,724 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T02:22:30,730 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T02:22:30,734 INFO [master/331c7316141f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T02:22:30,739 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T02:22:30,751 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T02:22:30,754 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T02:22:30,762 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T02:22:30,785 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T02:22:30,793 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T02:22:30,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T02:22:30,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T02:22:30,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:30,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:30,808 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=331c7316141f,41605,1731032547397, sessionid=0x101184078e70000, setting cluster-up flag (Was=false) 2024-11-08T02:22:30,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:30,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:30,878 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T02:22:30,881 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=331c7316141f,41605,1731032547397 2024-11-08T02:22:30,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:30,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:30,941 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T02:22:30,943 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=331c7316141f,41605,1731032547397 2024-11-08T02:22:30,950 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T02:22:31,034 INFO [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(746): ClusterId : ad3e781f-9d7b-4875-a9cb-20151d78056f 2024-11-08T02:22:31,037 DEBUG [RS:0;331c7316141f:34325 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T02:22:31,039 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T02:22:31,048 DEBUG [RS:0;331c7316141f:34325 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T02:22:31,048 DEBUG [RS:0;331c7316141f:34325 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T02:22:31,050 INFO [master/331c7316141f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T02:22:31,056 INFO [master/331c7316141f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T02:22:31,058 DEBUG [RS:0;331c7316141f:34325 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T02:22:31,059 DEBUG [RS:0;331c7316141f:34325 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e99c276, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=331c7316141f/172.17.0.2:0 2024-11-08T02:22:31,064 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 331c7316141f,41605,1731032547397 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T02:22:31,074 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/331c7316141f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T02:22:31,075 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/331c7316141f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T02:22:31,075 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/331c7316141f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T02:22:31,075 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/331c7316141f:0, corePoolSize=5, maxPoolSize=5 2024-11-08T02:22:31,076 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/331c7316141f:0, corePoolSize=10, maxPoolSize=10 2024-11-08T02:22:31,076 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,076 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/331c7316141f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T02:22:31,076 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,078 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;331c7316141f:34325 2024-11-08T02:22:31,081 INFO [RS:0;331c7316141f:34325 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T02:22:31,081 INFO [RS:0;331c7316141f:34325 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T02:22:31,082 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T02:22:31,085 INFO [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(2659): reportForDuty to master=331c7316141f,41605,1731032547397 with port=34325, startcode=1731032548147 2024-11-08T02:22:31,089 INFO [master/331c7316141f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731032581089 2024-11-08T02:22:31,091 INFO [master/331c7316141f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T02:22:31,093 INFO [master/331c7316141f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T02:22:31,097 INFO [master/331c7316141f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T02:22:31,098 INFO [master/331c7316141f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T02:22:31,098 INFO [master/331c7316141f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T02:22:31,099 INFO [master/331c7316141f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T02:22:31,099 DEBUG [RS:0;331c7316141f:34325 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T02:22:31,100 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T02:22:31,101 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T02:22:31,100 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,108 INFO [master/331c7316141f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T02:22:31,109 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:31,109 INFO [master/331c7316141f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T02:22:31,109 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T02:22:31,110 INFO [master/331c7316141f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T02:22:31,112 INFO [master/331c7316141f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T02:22:31,113 INFO [master/331c7316141f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T02:22:31,115 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/331c7316141f:0:becomeActiveMaster-HFileCleaner.large.0-1731032551114,5,FailOnTimeoutGroup] 2024-11-08T02:22:31,116 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/331c7316141f:0:becomeActiveMaster-HFileCleaner.small.0-1731032551116,5,FailOnTimeoutGroup] 2024-11-08T02:22:31,117 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,117 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T02:22:31,118 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,119 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741831_1007 (size=1321) 2024-11-08T02:22:31,135 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T02:22:31,136 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee 2024-11-08T02:22:31,146 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35013, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T02:22:31,155 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41605 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 331c7316141f,34325,1731032548147 2024-11-08T02:22:31,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741832_1008 (size=32) 2024-11-08T02:22:31,159 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41605 {}] master.ServerManager(517): Registering regionserver=331c7316141f,34325,1731032548147 2024-11-08T02:22:31,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:22:31,165 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T02:22:31,169 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T02:22:31,169 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:31,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:22:31,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T02:22:31,174 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T02:22:31,175 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:31,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:22:31,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T02:22:31,181 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee 2024-11-08T02:22:31,181 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43585 2024-11-08T02:22:31,181 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T02:22:31,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T02:22:31,183 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:31,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:22:31,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T02:22:31,189 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T02:22:31,190 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:31,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:22:31,192 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T02:22:31,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T02:22:31,195 DEBUG [RS:0;331c7316141f:34325 {}] zookeeper.ZKUtil(111): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/331c7316141f,34325,1731032548147 2024-11-08T02:22:31,195 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740 2024-11-08T02:22:31,195 WARN [RS:0;331c7316141f:34325 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T02:22:31,195 INFO [RS:0;331c7316141f:34325 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T02:22:31,196 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/WALs/331c7316141f,34325,1731032548147 2024-11-08T02:22:31,196 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740 2024-11-08T02:22:31,197 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [331c7316141f,34325,1731032548147] 2024-11-08T02:22:31,202 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T02:22:31,202 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T02:22:31,203 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:22:31,207 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T02:22:31,214 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:22:31,215 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71003832, jitterRate=0.05803954601287842}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:22:31,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731032551162Initializing all the Stores at 1731032551165 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731032551165Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731032551165Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032551165Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731032551165Cleaning up temporary data from old regions at 1731032551202 (+37 ms)Region opened successfully at 1731032551219 (+17 ms) 2024-11-08T02:22:31,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T02:22:31,220 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T02:22:31,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T02:22:31,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T02:22:31,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T02:22:31,221 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T02:22:31,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731032551220Disabling compacts and flushes for region at 1731032551220Disabling writes for close at 1731032551220Writing region close event to WAL at 1731032551221 (+1 ms)Closed at 1731032551221 2024-11-08T02:22:31,225 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T02:22:31,226 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T02:22:31,228 INFO [RS:0;331c7316141f:34325 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T02:22:31,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T02:22:31,244 INFO [RS:0;331c7316141f:34325 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T02:22:31,245 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T02:22:31,248 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T02:22:31,250 INFO [RS:0;331c7316141f:34325 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T02:22:31,250 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,251 INFO [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T02:22:31,256 INFO [RS:0;331c7316141f:34325 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T02:22:31,258 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,258 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,258 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,258 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,258 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,259 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,259 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/331c7316141f:0, corePoolSize=2, maxPoolSize=2 2024-11-08T02:22:31,259 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,259 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,259 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,259 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,259 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,260 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/331c7316141f:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:22:31,260 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/331c7316141f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T02:22:31,260 DEBUG [RS:0;331c7316141f:34325 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0, corePoolSize=3, maxPoolSize=3 2024-11-08T02:22:31,261 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,261 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,261 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,261 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,261 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,262 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(168): Chore ScheduledChore name=331c7316141f,34325,1731032548147-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T02:22:31,279 INFO [RS:0;331c7316141f:34325 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T02:22:31,281 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(168): Chore ScheduledChore name=331c7316141f,34325,1731032548147-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,281 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,281 INFO [RS:0;331c7316141f:34325 {}] regionserver.Replication(171): 331c7316141f,34325,1731032548147 started 2024-11-08T02:22:31,304 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:31,305 INFO [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1482): Serving as 331c7316141f,34325,1731032548147, RpcServer on 331c7316141f/172.17.0.2:34325, sessionid=0x101184078e70001 2024-11-08T02:22:31,306 DEBUG [RS:0;331c7316141f:34325 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T02:22:31,306 DEBUG [RS:0;331c7316141f:34325 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 331c7316141f,34325,1731032548147 2024-11-08T02:22:31,306 DEBUG [RS:0;331c7316141f:34325 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '331c7316141f,34325,1731032548147' 2024-11-08T02:22:31,306 DEBUG [RS:0;331c7316141f:34325 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T02:22:31,307 DEBUG [RS:0;331c7316141f:34325 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T02:22:31,308 DEBUG [RS:0;331c7316141f:34325 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T02:22:31,308 DEBUG [RS:0;331c7316141f:34325 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T02:22:31,309 DEBUG [RS:0;331c7316141f:34325 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 331c7316141f,34325,1731032548147 2024-11-08T02:22:31,309 DEBUG [RS:0;331c7316141f:34325 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '331c7316141f,34325,1731032548147' 2024-11-08T02:22:31,309 DEBUG [RS:0;331c7316141f:34325 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T02:22:31,309 DEBUG [RS:0;331c7316141f:34325 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T02:22:31,310 DEBUG [RS:0;331c7316141f:34325 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T02:22:31,310 INFO [RS:0;331c7316141f:34325 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T02:22:31,311 INFO [RS:0;331c7316141f:34325 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T02:22:31,400 WARN [331c7316141f:41605 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-08T02:22:31,415 INFO [RS:0;331c7316141f:34325 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T02:22:31,419 INFO [RS:0;331c7316141f:34325 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=331c7316141f%2C34325%2C1731032548147, suffix=, logDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/WALs/331c7316141f,34325,1731032548147, archiveDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/oldWALs, maxLogs=32 2024-11-08T02:22:31,434 DEBUG [RS:0;331c7316141f:34325 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/WALs/331c7316141f,34325,1731032548147/331c7316141f%2C34325%2C1731032548147.1731032551421, exclude list is [], retry=0 2024-11-08T02:22:31,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40173,DS-3afd9e80-6193-4ae9-ba38-1bba3b9e56eb,DISK] 2024-11-08T02:22:31,443 INFO [RS:0;331c7316141f:34325 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/WALs/331c7316141f,34325,1731032548147/331c7316141f%2C34325%2C1731032548147.1731032551421 2024-11-08T02:22:31,443 DEBUG [RS:0;331c7316141f:34325 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34541:34541)] 2024-11-08T02:22:31,653 DEBUG [331c7316141f:41605 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T02:22:31,666 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:22:31,673 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 331c7316141f,34325,1731032548147, state=OPENING 2024-11-08T02:22:31,698 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T02:22:31,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:31,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:22:31,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T02:22:31,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T02:22:31,711 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T02:22:31,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:22:31,890 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T02:22:31,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32783, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T02:22:31,907 INFO [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T02:22:31,907 INFO [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T02:22:31,908 INFO [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T02:22:31,912 INFO [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=331c7316141f%2C34325%2C1731032548147.meta, suffix=.meta, logDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/WALs/331c7316141f,34325,1731032548147, archiveDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/oldWALs, maxLogs=32 2024-11-08T02:22:31,936 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/WALs/331c7316141f,34325,1731032548147/331c7316141f%2C34325%2C1731032548147.meta.1731032551915.meta, exclude list is [], retry=0 2024-11-08T02:22:31,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40173,DS-3afd9e80-6193-4ae9-ba38-1bba3b9e56eb,DISK] 2024-11-08T02:22:31,944 INFO [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/WALs/331c7316141f,34325,1731032548147/331c7316141f%2C34325%2C1731032548147.meta.1731032551915.meta 2024-11-08T02:22:31,945 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34541:34541)] 2024-11-08T02:22:31,945 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:22:31,948 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T02:22:31,951 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T02:22:31,956 INFO [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T02:22:31,961 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T02:22:31,962 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:22:31,962 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T02:22:31,962 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T02:22:31,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T02:22:31,968 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T02:22:31,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:31,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:22:31,970 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T02:22:31,972 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T02:22:31,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:31,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:22:31,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T02:22:31,976 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T02:22:31,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:31,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:22:31,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T02:22:31,979 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T02:22:31,979 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:31,980 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:22:31,981 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T02:22:31,983 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740 2024-11-08T02:22:31,986 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740 2024-11-08T02:22:31,989 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T02:22:31,989 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T02:22:31,990 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:22:31,993 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T02:22:31,995 INFO [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70222681, jitterRate=0.04639948904514313}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:22:31,995 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T02:22:31,997 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731032551963Writing region info on filesystem at 1731032551963Initializing all the Stores at 1731032551966 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731032551966Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731032551966Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032551966Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731032551966Cleaning up temporary data from old regions at 1731032551989 (+23 ms)Running coprocessor post-open hooks at 1731032551995 (+6 ms)Region opened successfully at 1731032551996 (+1 ms) 2024-11-08T02:22:32,003 INFO [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731032551881 2024-11-08T02:22:32,016 DEBUG [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T02:22:32,017 INFO [RS_OPEN_META-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T02:22:32,019 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:22:32,022 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 331c7316141f,34325,1731032548147, state=OPEN 2024-11-08T02:22:32,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T02:22:32,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T02:22:32,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T02:22:32,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T02:22:32,085 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:32,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T02:22:32,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=331c7316141f,34325,1731032548147 in 372 msec 2024-11-08T02:22:32,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T02:22:32,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 861 msec 2024-11-08T02:22:32,104 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T02:22:32,104 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T02:22:32,129 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:32,131 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:32,158 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:32,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51289, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:32,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2010 sec 2024-11-08T02:22:32,189 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731032552189, completionTime=-1 2024-11-08T02:22:32,193 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-08T02:22:32,194 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T02:22:32,231 INFO [master/331c7316141f:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-08T02:22:32,231 INFO [master/331c7316141f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731032612231 2024-11-08T02:22:32,232 INFO [master/331c7316141f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731032672231 2024-11-08T02:22:32,232 INFO [master/331c7316141f:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 38 msec 2024-11-08T02:22:32,235 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=331c7316141f,41605,1731032547397-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:32,235 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=331c7316141f,41605,1731032547397-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:32,235 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=331c7316141f,41605,1731032547397-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:32,238 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-331c7316141f:41605, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:32,238 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:32,239 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:32,247 DEBUG [master/331c7316141f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T02:22:32,273 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.973sec 2024-11-08T02:22:32,274 INFO [master/331c7316141f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T02:22:32,276 INFO [master/331c7316141f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T02:22:32,277 INFO [master/331c7316141f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T02:22:32,277 INFO [master/331c7316141f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T02:22:32,277 INFO [master/331c7316141f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T02:22:32,278 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=331c7316141f,41605,1731032547397-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T02:22:32,278 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=331c7316141f,41605,1731032547397-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T02:22:32,288 DEBUG [master/331c7316141f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T02:22:32,289 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T02:22:32,289 INFO [master/331c7316141f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=331c7316141f,41605,1731032547397-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T02:22:32,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:32,350 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T02:22:32,350 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T02:22:32,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:22:32,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:22:32,370 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:22:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:22:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:22:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:22:32,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:22:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:32,381 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:22:32,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:32,385 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:32,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:32,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:32,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:32,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=331c7316141f,41605,1731032547397 2024-11-08T02:22:32,432 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=417, ProcessCount=11, AvailableMemoryMB=8294 2024-11-08T02:22:32,466 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T02:22:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 331c7316141f,41605,1731032547397 2024-11-08T02:22:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56ef0928 2024-11-08T02:22:32,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:22:32,480 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:22:32,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:22:32,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:22:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:22:32,499 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:22:32,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-08T02:22:32,501 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:32,503 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:22:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T02:22:32,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741835_1011 (size=959) 2024-11-08T02:22:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T02:22:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T02:22:32,919 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee 2024-11-08T02:22:32,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741836_1012 (size=53) 2024-11-08T02:22:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T02:22:33,335 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:22:33,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8469534ae513a00a18db97d665b015cd, disabling compactions & flushes 2024-11-08T02:22:33,336 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:33,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:33,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. after waiting 0 ms 2024-11-08T02:22:33,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:33,336 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:33,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8469534ae513a00a18db97d665b015cd: Waiting for close lock at 1731032553336Disabling compacts and flushes for region at 1731032553336Disabling writes for close at 1731032553336Writing region close event to WAL at 1731032553336Closed at 1731032553336 2024-11-08T02:22:33,339 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:22:33,345 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731032553339"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731032553339"}]},"ts":"1731032553339"} 2024-11-08T02:22:33,351 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T02:22:33,354 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:22:33,357 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032553354"}]},"ts":"1731032553354"} 2024-11-08T02:22:33,364 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:22:33,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8469534ae513a00a18db97d665b015cd, ASSIGN}] 2024-11-08T02:22:33,370 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8469534ae513a00a18db97d665b015cd, ASSIGN 2024-11-08T02:22:33,373 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8469534ae513a00a18db97d665b015cd, ASSIGN; state=OFFLINE, location=331c7316141f,34325,1731032548147; forceNewPlan=false, retain=false 2024-11-08T02:22:33,525 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8469534ae513a00a18db97d665b015cd, regionState=OPENING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:22:33,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8469534ae513a00a18db97d665b015cd, ASSIGN because future has completed 2024-11-08T02:22:33,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:22:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T02:22:33,700 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:33,700 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:22:33,701 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,701 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:22:33,702 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,702 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,704 INFO [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,707 INFO [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8469534ae513a00a18db97d665b015cd columnFamilyName A 2024-11-08T02:22:33,707 DEBUG [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:33,708 INFO [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] regionserver.HStore(327): Store=8469534ae513a00a18db97d665b015cd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:22:33,708 INFO [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,710 INFO [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8469534ae513a00a18db97d665b015cd columnFamilyName B 2024-11-08T02:22:33,710 DEBUG [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:33,712 INFO [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] regionserver.HStore(327): Store=8469534ae513a00a18db97d665b015cd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:22:33,712 INFO [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,715 INFO [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8469534ae513a00a18db97d665b015cd columnFamilyName C 2024-11-08T02:22:33,715 DEBUG [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:22:33,715 INFO [StoreOpener-8469534ae513a00a18db97d665b015cd-1 {}] regionserver.HStore(327): Store=8469534ae513a00a18db97d665b015cd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:22:33,716 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,717 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,718 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,721 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,721 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,722 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:22:33,725 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,728 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:22:33,729 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8469534ae513a00a18db97d665b015cd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65115968, jitterRate=-0.02969646453857422}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:22:33,729 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:33,730 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8469534ae513a00a18db97d665b015cd: Running coprocessor pre-open hook at 1731032553702Writing region info on filesystem at 1731032553702Initializing all the Stores at 1731032553704 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032553704Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032553704Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032553704Cleaning up temporary data from old regions at 1731032553721 (+17 ms)Running coprocessor post-open hooks at 1731032553729 (+8 ms)Region opened successfully at 1731032553730 (+1 ms) 2024-11-08T02:22:33,732 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., pid=6, masterSystemTime=1731032553692 2024-11-08T02:22:33,735 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:33,735 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:33,736 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8469534ae513a00a18db97d665b015cd, regionState=OPEN, openSeqNum=2, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:22:33,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:22:33,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T02:22:33,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 in 207 msec 2024-11-08T02:22:33,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T02:22:33,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8469534ae513a00a18db97d665b015cd, ASSIGN in 381 msec 2024-11-08T02:22:33,756 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:22:33,757 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032553757"}]},"ts":"1731032553757"} 2024-11-08T02:22:33,760 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:22:33,762 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:22:33,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2700 sec 2024-11-08T02:22:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T02:22:34,660 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:22:34,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:22:34,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:22:34,669 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:22:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:22:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:22:34,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56cc3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:22:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:22:34,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:34,677 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:22:34,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d59d180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,681 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:34,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:34,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:34,694 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T02:22:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 331c7316141f,41605,1731032547397 2024-11-08T02:22:34,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5147d04a 2024-11-08T02:22:34,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:22:34,704 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:22:34,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,709 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:22:34,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:22:34,713 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:22:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:22:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:22:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b147913, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:22:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:22:34,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:34,717 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:22:34,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@408cf13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:22:34,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:22:34,725 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:22:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:22:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:22:34,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ddcdb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:22:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:22:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:34,731 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:22:34,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e463f6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:22:34,737 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:22:34,738 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:22:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:22:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:22:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355546b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:22:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:22:34,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:34,742 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:22:34,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4c56da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:22:34,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:22:34,747 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:22:34,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:22:34,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:22:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:22:34,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:22:34,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:34,752 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:22:34,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1850bf91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:22:34,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:22:34,758 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:22:34,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:22:34,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:22:34,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:22:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:22:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:34,762 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:22:34,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25318c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:22:34,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:22:34,768 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:22:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:22:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:22:34,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:22:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:22:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:34,772 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:22:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129161b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:22:34,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:22:34,776 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:22:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:22:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:22:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac65b3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:22:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:22:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:34,779 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:22:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79314f91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:22:34,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:22:34,786 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:22:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:22:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:22:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a16c6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:22:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:22:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:34,790 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:22:34,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef2cbdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:22:34,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:22:34,794 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:22:34,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:22:34,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:22:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a4abfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:22:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:22:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:34,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:22:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1556c2b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:22:34,805 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:34,805 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:34,806 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:34,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:34,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:34,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:34,811 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:34,812 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:34,815 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:34,815 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:34,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:34,817 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:34,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:34,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:34,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:22:34,821 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:34,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:22:34,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:34,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:34,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:34,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:22:34,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:22:34,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:34,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:22:34,834 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:34,836 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:34,837 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:22:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:22:34,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:34,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:22:34,849 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:22:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:22:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:22:34,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:22:34,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:22:34,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:22:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-08T02:22:34,882 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:22:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T02:22:34,884 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:22:34,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:22:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:34,928 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:34,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:22:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T02:22:35,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:35,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:35,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/7aac23f2d00a4d91b607113ed6657f32 is 50, key is test_row_0/A:col10/1731032554929/Put/seqid=0 2024-11-08T02:22:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741837_1013 (size=14341) 2024-11-08T02:22:35,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/7aac23f2d00a4d91b607113ed6657f32 2024-11-08T02:22:35,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032615134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032615138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032615145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032615143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032615148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T02:22:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:35,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:35,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032615329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032615329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032615331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032615331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/205b5812c98b4d3d92f5b2991bcd8d6f is 50, key is test_row_0/B:col10/1731032554929/Put/seqid=0 2024-11-08T02:22:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032615331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741838_1014 (size=12001) 2024-11-08T02:22:35,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/205b5812c98b4d3d92f5b2991bcd8d6f 2024-11-08T02:22:35,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:35,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/51670cba9b9b44c596d59e1551042eb5 is 50, key is test_row_0/C:col10/1731032554929/Put/seqid=0 2024-11-08T02:22:35,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741839_1015 (size=12001) 2024-11-08T02:22:35,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/51670cba9b9b44c596d59e1551042eb5 2024-11-08T02:22:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T02:22:35,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/7aac23f2d00a4d91b607113ed6657f32 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7aac23f2d00a4d91b607113ed6657f32 2024-11-08T02:22:35,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7aac23f2d00a4d91b607113ed6657f32, entries=200, sequenceid=13, filesize=14.0 K 2024-11-08T02:22:35,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/205b5812c98b4d3d92f5b2991bcd8d6f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/205b5812c98b4d3d92f5b2991bcd8d6f 2024-11-08T02:22:35,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/205b5812c98b4d3d92f5b2991bcd8d6f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T02:22:35,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/51670cba9b9b44c596d59e1551042eb5 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/51670cba9b9b44c596d59e1551042eb5 2024-11-08T02:22:35,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032615549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032615549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/51670cba9b9b44c596d59e1551042eb5, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T02:22:35,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:35,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:35,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8469534ae513a00a18db97d665b015cd in 671ms, sequenceid=13, compaction requested=false 2024-11-08T02:22:35,616 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T02:22:35,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:35,651 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:35,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-08T02:22:35,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/d88ef76fb40d4347b7b7842bdbd33e9d is 50, key is test_row_0/A:col10/1731032555621/Put/seqid=0 2024-11-08T02:22:35,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032615672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032615676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032615679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741840_1016 (size=12001) 2024-11-08T02:22:35,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/d88ef76fb40d4347b7b7842bdbd33e9d 2024-11-08T02:22:35,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/037afbce69b54f579c40f09adccd730b is 50, key is test_row_0/B:col10/1731032555621/Put/seqid=0 2024-11-08T02:22:35,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741841_1017 (size=12001) 2024-11-08T02:22:35,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/037afbce69b54f579c40f09adccd730b 2024-11-08T02:22:35,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:35,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:35,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032615799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032615799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/b61047686df94aefbe008073647d86e1 is 50, key is test_row_0/C:col10/1731032555621/Put/seqid=0 2024-11-08T02:22:35,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032615809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741842_1018 (size=12001) 2024-11-08T02:22:35,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/b61047686df94aefbe008073647d86e1 2024-11-08T02:22:35,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/d88ef76fb40d4347b7b7842bdbd33e9d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d88ef76fb40d4347b7b7842bdbd33e9d 2024-11-08T02:22:35,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032615879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032615889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:35,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:35,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d88ef76fb40d4347b7b7842bdbd33e9d, entries=150, sequenceid=43, filesize=11.7 K 2024-11-08T02:22:35,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/037afbce69b54f579c40f09adccd730b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/037afbce69b54f579c40f09adccd730b 2024-11-08T02:22:35,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:35,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:35,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:35,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:35,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/037afbce69b54f579c40f09adccd730b, entries=150, sequenceid=43, filesize=11.7 K 2024-11-08T02:22:35,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/b61047686df94aefbe008073647d86e1 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b61047686df94aefbe008073647d86e1 2024-11-08T02:22:35,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b61047686df94aefbe008073647d86e1, entries=150, sequenceid=43, filesize=11.7 K 2024-11-08T02:22:35,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 8469534ae513a00a18db97d665b015cd in 292ms, sequenceid=43, compaction requested=false 2024-11-08T02:22:35,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T02:22:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:36,033 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:36,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:22:36,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/54867e182d3e4ffb9a688ae169aeac0d is 50, key is test_row_0/A:col10/1731032556025/Put/seqid=0 2024-11-08T02:22:36,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:36,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:36,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741843_1019 (size=16681) 2024-11-08T02:22:36,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032616186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032616190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032616197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032616310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032616319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032616320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:36,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:36,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032616399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032616408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/54867e182d3e4ffb9a688ae169aeac0d 2024-11-08T02:22:36,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032616519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/ec9d773296334b34b887917c324006ea is 50, key is test_row_0/B:col10/1731032556025/Put/seqid=0 2024-11-08T02:22:36,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741844_1020 (size=12001) 2024-11-08T02:22:36,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032616541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032616541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:36,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:36,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:36,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:36,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032616830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032616859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032616859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:36,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:36,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:36,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:36,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:36,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/ec9d773296334b34b887917c324006ea 2024-11-08T02:22:36,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/22346e163c884a8e8f3e888939c4dfe8 is 50, key is test_row_0/C:col10/1731032556025/Put/seqid=0 2024-11-08T02:22:36,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741845_1021 (size=12001) 2024-11-08T02:22:37,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/22346e163c884a8e8f3e888939c4dfe8 2024-11-08T02:22:37,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/54867e182d3e4ffb9a688ae169aeac0d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/54867e182d3e4ffb9a688ae169aeac0d 2024-11-08T02:22:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T02:22:37,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:37,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:37,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:37,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/54867e182d3e4ffb9a688ae169aeac0d, entries=250, sequenceid=54, filesize=16.3 K 2024-11-08T02:22:37,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/ec9d773296334b34b887917c324006ea as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec9d773296334b34b887917c324006ea 2024-11-08T02:22:37,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec9d773296334b34b887917c324006ea, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T02:22:37,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/22346e163c884a8e8f3e888939c4dfe8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/22346e163c884a8e8f3e888939c4dfe8 2024-11-08T02:22:37,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/22346e163c884a8e8f3e888939c4dfe8, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T02:22:37,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8469534ae513a00a18db97d665b015cd in 1066ms, sequenceid=54, compaction requested=true 2024-11-08T02:22:37,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:37,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:37,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:37,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:37,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:37,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:37,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:37,114 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:37,114 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:37,119 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:37,119 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:37,121 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:37,122 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:37,123 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/205b5812c98b4d3d92f5b2991bcd8d6f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/037afbce69b54f579c40f09adccd730b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec9d773296334b34b887917c324006ea] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=35.2 K 2024-11-08T02:22:37,124 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:37,125 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:37,125 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 205b5812c98b4d3d92f5b2991bcd8d6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731032554878 2024-11-08T02:22:37,126 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7aac23f2d00a4d91b607113ed6657f32, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d88ef76fb40d4347b7b7842bdbd33e9d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/54867e182d3e4ffb9a688ae169aeac0d] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=42.0 K 2024-11-08T02:22:37,126 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 037afbce69b54f579c40f09adccd730b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731032555601 2024-11-08T02:22:37,127 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting ec9d773296334b34b887917c324006ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032556015 2024-11-08T02:22:37,127 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aac23f2d00a4d91b607113ed6657f32, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731032554878 2024-11-08T02:22:37,130 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting d88ef76fb40d4347b7b7842bdbd33e9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731032555601 2024-11-08T02:22:37,134 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54867e182d3e4ffb9a688ae169aeac0d, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032555674 2024-11-08T02:22:37,183 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#9 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:37,183 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:37,184 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/d0ebfd02ef4e4c19a8d95f412097171a is 50, key is test_row_0/B:col10/1731032556025/Put/seqid=0 2024-11-08T02:22:37,184 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/9ca585a1529147439680c4313d55f658 is 50, key is test_row_0/A:col10/1731032556025/Put/seqid=0 2024-11-08T02:22:37,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T02:22:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:37,204 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:22:37,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/fb94f483c980415f982d000902b7f85f is 50, key is test_row_0/A:col10/1731032556187/Put/seqid=0 2024-11-08T02:22:37,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741847_1023 (size=12104) 2024-11-08T02:22:37,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741846_1022 (size=12104) 2024-11-08T02:22:37,276 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/9ca585a1529147439680c4313d55f658 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9ca585a1529147439680c4313d55f658 2024-11-08T02:22:37,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741848_1024 (size=12001) 2024-11-08T02:22:37,288 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/fb94f483c980415f982d000902b7f85f 2024-11-08T02:22:37,302 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into 9ca585a1529147439680c4313d55f658(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:37,303 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:37,306 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=13, startTime=1731032557101; duration=0sec 2024-11-08T02:22:37,307 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:37,307 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:37,307 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:37,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/d960a4b6a2fe46928b55b1ff03f8f533 is 50, key is test_row_0/B:col10/1731032556187/Put/seqid=0 2024-11-08T02:22:37,330 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:37,330 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:37,330 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:37,330 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/51670cba9b9b44c596d59e1551042eb5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b61047686df94aefbe008073647d86e1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/22346e163c884a8e8f3e888939c4dfe8] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=35.2 K 2024-11-08T02:22:37,332 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51670cba9b9b44c596d59e1551042eb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731032554878 2024-11-08T02:22:37,333 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b61047686df94aefbe008073647d86e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731032555601 2024-11-08T02:22:37,335 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22346e163c884a8e8f3e888939c4dfe8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032556015 2024-11-08T02:22:37,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741849_1025 (size=12001) 2024-11-08T02:22:37,360 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/d960a4b6a2fe46928b55b1ff03f8f533 2024-11-08T02:22:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:37,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:37,387 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#13 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:37,388 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/2875c5e6db024005b8b5788f0ece11f1 is 50, key is test_row_0/C:col10/1731032556025/Put/seqid=0 2024-11-08T02:22:37,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/56ff89a8deee4ac0a8b492a230e85ab5 is 50, key is test_row_0/C:col10/1731032556187/Put/seqid=0 2024-11-08T02:22:37,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741850_1026 (size=12104) 2024-11-08T02:22:37,439 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/2875c5e6db024005b8b5788f0ece11f1 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/2875c5e6db024005b8b5788f0ece11f1 2024-11-08T02:22:37,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741851_1027 (size=12001) 2024-11-08T02:22:37,444 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/56ff89a8deee4ac0a8b492a230e85ab5 2024-11-08T02:22:37,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032617431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032617436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032617436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032617442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032617444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,467 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into 2875c5e6db024005b8b5788f0ece11f1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:37,467 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:37,467 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=13, startTime=1731032557111; duration=0sec 2024-11-08T02:22:37,467 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:37,467 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:37,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/fb94f483c980415f982d000902b7f85f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fb94f483c980415f982d000902b7f85f 2024-11-08T02:22:37,478 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:22:37,485 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fb94f483c980415f982d000902b7f85f, entries=150, sequenceid=79, filesize=11.7 K 2024-11-08T02:22:37,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/d960a4b6a2fe46928b55b1ff03f8f533 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d960a4b6a2fe46928b55b1ff03f8f533 2024-11-08T02:22:37,508 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d960a4b6a2fe46928b55b1ff03f8f533, entries=150, sequenceid=79, filesize=11.7 K 2024-11-08T02:22:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/56ff89a8deee4ac0a8b492a230e85ab5 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/56ff89a8deee4ac0a8b492a230e85ab5 2024-11-08T02:22:37,527 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/56ff89a8deee4ac0a8b492a230e85ab5, entries=150, sequenceid=79, filesize=11.7 K 2024-11-08T02:22:37,529 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8469534ae513a00a18db97d665b015cd in 324ms, sequenceid=79, compaction requested=false 2024-11-08T02:22:37,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:37,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:37,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-08T02:22:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-08T02:22:37,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T02:22:37,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6490 sec 2024-11-08T02:22:37,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.6680 sec 2024-11-08T02:22:37,575 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:37,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T02:22:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:37,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/7d2c545870c547da932d151c0260d391 is 50, key is test_row_0/A:col10/1731032557569/Put/seqid=0 2024-11-08T02:22:37,597 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T02:22:37,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741852_1028 (size=16681) 2024-11-08T02:22:37,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/7d2c545870c547da932d151c0260d391 2024-11-08T02:22:37,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/590f4f45faa6453688a5ed34b4b9a51b is 50, key is test_row_0/B:col10/1731032557569/Put/seqid=0 2024-11-08T02:22:37,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032617649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032617655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032617652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,672 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/d0ebfd02ef4e4c19a8d95f412097171a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d0ebfd02ef4e4c19a8d95f412097171a 2024-11-08T02:22:37,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032617666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032617664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,690 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into d0ebfd02ef4e4c19a8d95f412097171a(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:37,690 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:37,690 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=13, startTime=1731032557111; duration=0sec 2024-11-08T02:22:37,691 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:37,691 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:37,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741853_1029 (size=12001) 2024-11-08T02:22:37,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/590f4f45faa6453688a5ed34b4b9a51b 2024-11-08T02:22:37,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/6b4aabedcce94c40b85b1fe3e76415ad is 50, key is test_row_0/C:col10/1731032557569/Put/seqid=0 2024-11-08T02:22:37,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741854_1030 (size=12001) 2024-11-08T02:22:37,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/6b4aabedcce94c40b85b1fe3e76415ad 2024-11-08T02:22:37,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T02:22:37,772 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T02:22:37,775 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T02:22:37,775 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T02:22:37,776 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T02:22:37,777 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T02:22:37,778 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T02:22:37,778 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-08T02:22:37,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/7d2c545870c547da932d151c0260d391 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7d2c545870c547da932d151c0260d391 2024-11-08T02:22:37,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032617779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032617780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032617780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032617779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032617780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:37,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:37,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7d2c545870c547da932d151c0260d391, entries=250, sequenceid=94, filesize=16.3 K 2024-11-08T02:22:37,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/590f4f45faa6453688a5ed34b4b9a51b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/590f4f45faa6453688a5ed34b4b9a51b 2024-11-08T02:22:37,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/590f4f45faa6453688a5ed34b4b9a51b, entries=150, sequenceid=94, filesize=11.7 K 2024-11-08T02:22:37,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/6b4aabedcce94c40b85b1fe3e76415ad as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/6b4aabedcce94c40b85b1fe3e76415ad 2024-11-08T02:22:37,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/6b4aabedcce94c40b85b1fe3e76415ad, entries=150, sequenceid=94, filesize=11.7 K 2024-11-08T02:22:37,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 8469534ae513a00a18db97d665b015cd in 279ms, sequenceid=94, compaction requested=true 2024-11-08T02:22:37,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:37,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:37,855 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:37,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:37,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:37,855 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:37,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:37,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:37,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:37,860 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:37,860 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:37,860 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:37,860 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d0ebfd02ef4e4c19a8d95f412097171a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d960a4b6a2fe46928b55b1ff03f8f533, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/590f4f45faa6453688a5ed34b4b9a51b] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=35.3 K 2024-11-08T02:22:37,861 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:37,861 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:37,861 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:37,861 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9ca585a1529147439680c4313d55f658, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fb94f483c980415f982d000902b7f85f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7d2c545870c547da932d151c0260d391] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=39.8 K 2024-11-08T02:22:37,862 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ca585a1529147439680c4313d55f658, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032556015 2024-11-08T02:22:37,863 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting d0ebfd02ef4e4c19a8d95f412097171a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032556015 2024-11-08T02:22:37,864 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb94f483c980415f982d000902b7f85f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731032556179 2024-11-08T02:22:37,864 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting d960a4b6a2fe46928b55b1ff03f8f533, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731032556179 2024-11-08T02:22:37,865 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 590f4f45faa6453688a5ed34b4b9a51b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731032557437 2024-11-08T02:22:37,865 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d2c545870c547da932d151c0260d391, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731032557431 2024-11-08T02:22:37,899 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:37,900 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/9f4ee173de6444279ab59235f1ee4a51 is 50, key is test_row_0/A:col10/1731032557569/Put/seqid=0 2024-11-08T02:22:37,912 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:37,913 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/30f830fe10b84282817a3fc8e76ef9e6 is 50, key is test_row_0/B:col10/1731032557569/Put/seqid=0 2024-11-08T02:22:37,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741855_1031 (size=12207) 2024-11-08T02:22:37,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741856_1032 (size=12207) 2024-11-08T02:22:37,962 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/9f4ee173de6444279ab59235f1ee4a51 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9f4ee173de6444279ab59235f1ee4a51 2024-11-08T02:22:37,968 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/30f830fe10b84282817a3fc8e76ef9e6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/30f830fe10b84282817a3fc8e76ef9e6 2024-11-08T02:22:37,984 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into 30f830fe10b84282817a3fc8e76ef9e6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:37,984 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:37,984 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=13, startTime=1731032557855; duration=0sec 2024-11-08T02:22:37,985 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:37,985 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:37,985 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:37,987 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:37,987 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:37,988 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:37,988 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/2875c5e6db024005b8b5788f0ece11f1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/56ff89a8deee4ac0a8b492a230e85ab5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/6b4aabedcce94c40b85b1fe3e76415ad] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=35.3 K 2024-11-08T02:22:37,988 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into 9f4ee173de6444279ab59235f1ee4a51(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:37,988 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:37,989 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 2875c5e6db024005b8b5788f0ece11f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032556015 2024-11-08T02:22:37,989 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=13, startTime=1731032557854; duration=0sec 2024-11-08T02:22:37,989 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:37,990 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ff89a8deee4ac0a8b492a230e85ab5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731032556179 2024-11-08T02:22:37,991 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b4aabedcce94c40b85b1fe3e76415ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731032557437 2024-11-08T02:22:37,990 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:38,012 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:38,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:22:38,017 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:38,018 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/445b3032a2744dc4bd11791dc441a8be is 50, key is test_row_0/C:col10/1731032557569/Put/seqid=0 2024-11-08T02:22:38,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/79129cdadc8046fea9c26c49b78a6959 is 50, key is test_row_0/A:col10/1731032558008/Put/seqid=0 2024-11-08T02:22:38,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032618025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032618044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032618047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032618050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741857_1033 (size=12207) 2024-11-08T02:22:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032618053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741858_1034 (size=12001) 2024-11-08T02:22:38,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/79129cdadc8046fea9c26c49b78a6959 2024-11-08T02:22:38,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/01c10b8616bf434bb304a275b23ed402 is 50, key is test_row_0/B:col10/1731032558008/Put/seqid=0 2024-11-08T02:22:38,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032618158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741859_1035 (size=12001) 2024-11-08T02:22:38,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/01c10b8616bf434bb304a275b23ed402 2024-11-08T02:22:38,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032618169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032618169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032618169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032618179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/0434953c2e18421fa117f4982ba7cf46 is 50, key is test_row_0/C:col10/1731032558008/Put/seqid=0 2024-11-08T02:22:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741860_1036 (size=12001) 2024-11-08T02:22:38,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/0434953c2e18421fa117f4982ba7cf46 2024-11-08T02:22:38,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/79129cdadc8046fea9c26c49b78a6959 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/79129cdadc8046fea9c26c49b78a6959 2024-11-08T02:22:38,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/79129cdadc8046fea9c26c49b78a6959, entries=150, sequenceid=124, filesize=11.7 K 2024-11-08T02:22:38,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/01c10b8616bf434bb304a275b23ed402 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/01c10b8616bf434bb304a275b23ed402 2024-11-08T02:22:38,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/01c10b8616bf434bb304a275b23ed402, entries=150, sequenceid=124, filesize=11.7 K 2024-11-08T02:22:38,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/0434953c2e18421fa117f4982ba7cf46 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/0434953c2e18421fa117f4982ba7cf46 2024-11-08T02:22:38,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/0434953c2e18421fa117f4982ba7cf46, entries=150, sequenceid=124, filesize=11.7 K 2024-11-08T02:22:38,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 8469534ae513a00a18db97d665b015cd in 303ms, sequenceid=124, compaction requested=false 2024-11-08T02:22:38,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:38,381 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:38,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:22:38,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/a2c2c41dc9a444e5abb192bfff0b2cc0 is 50, key is test_row_0/A:col10/1731032558369/Put/seqid=0 2024-11-08T02:22:38,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741861_1037 (size=14541) 2024-11-08T02:22:38,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/a2c2c41dc9a444e5abb192bfff0b2cc0 2024-11-08T02:22:38,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/d268b491df044fbb9d7f3424ff407384 is 50, key is test_row_0/B:col10/1731032558369/Put/seqid=0 2024-11-08T02:22:38,473 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/445b3032a2744dc4bd11791dc441a8be as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/445b3032a2744dc4bd11791dc441a8be 2024-11-08T02:22:38,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032618465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032618469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032618472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,488 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into 445b3032a2744dc4bd11791dc441a8be(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:38,488 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:38,489 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=13, startTime=1731032557858; duration=0sec 2024-11-08T02:22:38,489 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:38,489 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:38,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032618476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032618477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741862_1038 (size=12151) 2024-11-08T02:22:38,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/d268b491df044fbb9d7f3424ff407384 2024-11-08T02:22:38,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/1f9ecb8aa63941b6866c644d6714fb16 is 50, key is test_row_0/C:col10/1731032558369/Put/seqid=0 2024-11-08T02:22:38,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741863_1039 (size=12151) 2024-11-08T02:22:38,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/1f9ecb8aa63941b6866c644d6714fb16 2024-11-08T02:22:38,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/a2c2c41dc9a444e5abb192bfff0b2cc0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/a2c2c41dc9a444e5abb192bfff0b2cc0 2024-11-08T02:22:38,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032618589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032618589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032618589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/a2c2c41dc9a444e5abb192bfff0b2cc0, entries=200, sequenceid=135, filesize=14.2 K 2024-11-08T02:22:38,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/d268b491df044fbb9d7f3424ff407384 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d268b491df044fbb9d7f3424ff407384 2024-11-08T02:22:38,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032618598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032618599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d268b491df044fbb9d7f3424ff407384, entries=150, sequenceid=135, filesize=11.9 K 2024-11-08T02:22:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/1f9ecb8aa63941b6866c644d6714fb16 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/1f9ecb8aa63941b6866c644d6714fb16 2024-11-08T02:22:38,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/1f9ecb8aa63941b6866c644d6714fb16, entries=150, sequenceid=135, filesize=11.9 K 2024-11-08T02:22:38,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8469534ae513a00a18db97d665b015cd in 249ms, sequenceid=135, compaction requested=true 2024-11-08T02:22:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:38,631 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:38,631 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:38,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:38,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:38,633 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:38,633 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:38,633 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:38,633 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:38,633 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:38,633 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:38,634 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9f4ee173de6444279ab59235f1ee4a51, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/79129cdadc8046fea9c26c49b78a6959, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/a2c2c41dc9a444e5abb192bfff0b2cc0] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=37.8 K 2024-11-08T02:22:38,634 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/30f830fe10b84282817a3fc8e76ef9e6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/01c10b8616bf434bb304a275b23ed402, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d268b491df044fbb9d7f3424ff407384] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=35.5 K 2024-11-08T02:22:38,635 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f4ee173de6444279ab59235f1ee4a51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731032557437 2024-11-08T02:22:38,635 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 30f830fe10b84282817a3fc8e76ef9e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731032557437 2024-11-08T02:22:38,637 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79129cdadc8046fea9c26c49b78a6959, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731032558008 2024-11-08T02:22:38,637 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 01c10b8616bf434bb304a275b23ed402, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731032558008 2024-11-08T02:22:38,638 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c2c41dc9a444e5abb192bfff0b2cc0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731032558046 2024-11-08T02:22:38,638 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting d268b491df044fbb9d7f3424ff407384, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731032558050 2024-11-08T02:22:38,667 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:38,670 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/e89b2708699d4f108082bd36737cb678 is 50, key is test_row_0/A:col10/1731032558369/Put/seqid=0 2024-11-08T02:22:38,675 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#28 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:38,676 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/0b02ed2c55df4909a26aae7bffe59dfa is 50, key is test_row_0/B:col10/1731032558369/Put/seqid=0 2024-11-08T02:22:38,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741864_1040 (size=12459) 2024-11-08T02:22:38,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741865_1041 (size=12459) 2024-11-08T02:22:38,714 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/e89b2708699d4f108082bd36737cb678 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e89b2708699d4f108082bd36737cb678 2024-11-08T02:22:38,730 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/0b02ed2c55df4909a26aae7bffe59dfa as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/0b02ed2c55df4909a26aae7bffe59dfa 2024-11-08T02:22:38,735 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into e89b2708699d4f108082bd36737cb678(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:38,735 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:38,735 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=13, startTime=1731032558631; duration=0sec 2024-11-08T02:22:38,735 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:38,735 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:38,735 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:38,739 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:38,739 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:38,739 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:38,741 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/445b3032a2744dc4bd11791dc441a8be, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/0434953c2e18421fa117f4982ba7cf46, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/1f9ecb8aa63941b6866c644d6714fb16] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=35.5 K 2024-11-08T02:22:38,742 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 445b3032a2744dc4bd11791dc441a8be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731032557437 2024-11-08T02:22:38,742 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0434953c2e18421fa117f4982ba7cf46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731032558008 2024-11-08T02:22:38,743 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f9ecb8aa63941b6866c644d6714fb16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731032558050 2024-11-08T02:22:38,765 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into 0b02ed2c55df4909a26aae7bffe59dfa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:38,765 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:38,765 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=13, startTime=1731032558631; duration=0sec 2024-11-08T02:22:38,765 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:38,766 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:38,794 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#29 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:38,795 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/e9df5b925325499bb8c145f3b6f9a96b is 50, key is test_row_0/C:col10/1731032558369/Put/seqid=0 2024-11-08T02:22:38,801 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:38,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:22:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:38,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/332f2b1131be4c318aa7f0bc354bd3bf is 50, key is test_row_0/A:col10/1731032558471/Put/seqid=0 2024-11-08T02:22:38,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741866_1042 (size=12459) 2024-11-08T02:22:38,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741867_1043 (size=14541) 2024-11-08T02:22:38,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032618816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032618818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032618858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032618858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032618859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032618987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032618987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032618988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:38,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032618987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T02:22:39,048 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:22:39,049 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:22:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-08T02:22:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T02:22:39,052 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:22:39,053 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:22:39,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:22:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T02:22:39,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032619170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032619199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032619199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032619199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T02:22:39,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:39,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:39,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:39,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032619209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/332f2b1131be4c318aa7f0bc354bd3bf 2024-11-08T02:22:39,247 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/e9df5b925325499bb8c145f3b6f9a96b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/e9df5b925325499bb8c145f3b6f9a96b 2024-11-08T02:22:39,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/718201a4bd104070b5ab690fb0792cd0 is 50, key is test_row_0/B:col10/1731032558471/Put/seqid=0 2024-11-08T02:22:39,262 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into e9df5b925325499bb8c145f3b6f9a96b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:39,262 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:39,262 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=13, startTime=1731032558631; duration=0sec 2024-11-08T02:22:39,262 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:39,262 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:39,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741868_1044 (size=12151) 2024-11-08T02:22:39,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/718201a4bd104070b5ab690fb0792cd0 2024-11-08T02:22:39,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/86e4eaed86664f84a19611cddaf82246 is 50, key is test_row_0/C:col10/1731032558471/Put/seqid=0 2024-11-08T02:22:39,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741869_1045 (size=12151) 2024-11-08T02:22:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T02:22:39,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T02:22:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:39,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032619508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:39,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032619509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032619509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032619521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T02:22:39,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:39,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:39,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:39,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T02:22:39,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032619680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:39,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T02:22:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:39,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:39,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:39,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:39,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:39,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/86e4eaed86664f84a19611cddaf82246 2024-11-08T02:22:39,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/332f2b1131be4c318aa7f0bc354bd3bf as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/332f2b1131be4c318aa7f0bc354bd3bf 2024-11-08T02:22:39,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/332f2b1131be4c318aa7f0bc354bd3bf, entries=200, sequenceid=164, filesize=14.2 K 2024-11-08T02:22:39,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/718201a4bd104070b5ab690fb0792cd0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/718201a4bd104070b5ab690fb0792cd0 2024-11-08T02:22:39,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/718201a4bd104070b5ab690fb0792cd0, entries=150, sequenceid=164, filesize=11.9 K 2024-11-08T02:22:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/86e4eaed86664f84a19611cddaf82246 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/86e4eaed86664f84a19611cddaf82246 2024-11-08T02:22:39,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/86e4eaed86664f84a19611cddaf82246, entries=150, sequenceid=164, filesize=11.9 K 2024-11-08T02:22:39,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 8469534ae513a00a18db97d665b015cd in 1028ms, sequenceid=164, compaction requested=false 2024-11-08T02:22:39,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:39,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T02:22:39,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:39,845 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:22:39,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/57661a2e566d49d48ecc52e9ecb339ff is 50, key is test_row_0/A:col10/1731032558816/Put/seqid=0 2024-11-08T02:22:39,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741870_1046 (size=12151) 2024-11-08T02:22:39,894 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/57661a2e566d49d48ecc52e9ecb339ff 2024-11-08T02:22:39,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/e15195eb7b4f4580b25ad3509013356a is 50, key is test_row_0/B:col10/1731032558816/Put/seqid=0 2024-11-08T02:22:39,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741871_1047 (size=12151) 2024-11-08T02:22:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:40,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:40,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032620070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032620070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032620076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032620077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032620189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032620189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T02:22:40,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032620198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032620199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,369 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/e15195eb7b4f4580b25ad3509013356a 2024-11-08T02:22:40,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/20a3ae5a19464281b9a47c40f38f01f8 is 50, key is test_row_0/C:col10/1731032558816/Put/seqid=0 2024-11-08T02:22:40,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032620399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032620399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032620408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032620408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741872_1048 (size=12151) 2024-11-08T02:22:40,445 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/20a3ae5a19464281b9a47c40f38f01f8 2024-11-08T02:22:40,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/57661a2e566d49d48ecc52e9ecb339ff as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/57661a2e566d49d48ecc52e9ecb339ff 2024-11-08T02:22:40,479 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/57661a2e566d49d48ecc52e9ecb339ff, entries=150, sequenceid=176, filesize=11.9 K 2024-11-08T02:22:40,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/e15195eb7b4f4580b25ad3509013356a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e15195eb7b4f4580b25ad3509013356a 2024-11-08T02:22:40,491 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e15195eb7b4f4580b25ad3509013356a, entries=150, sequenceid=176, filesize=11.9 K 2024-11-08T02:22:40,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/20a3ae5a19464281b9a47c40f38f01f8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/20a3ae5a19464281b9a47c40f38f01f8 2024-11-08T02:22:40,506 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/20a3ae5a19464281b9a47c40f38f01f8, entries=150, sequenceid=176, filesize=11.9 K 2024-11-08T02:22:40,507 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8469534ae513a00a18db97d665b015cd in 662ms, sequenceid=176, compaction requested=true 2024-11-08T02:22:40,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:40,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:40,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-08T02:22:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-08T02:22:40,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-08T02:22:40,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4560 sec 2024-11-08T02:22:40,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4660 sec 2024-11-08T02:22:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:40,701 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:40,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:22:40,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/2c99101b73bf433a990119f4de3c8d99 is 50, key is test_row_0/A:col10/1731032560699/Put/seqid=0 2024-11-08T02:22:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032620716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032620718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032620718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032620718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032620720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741873_1049 (size=14541) 2024-11-08T02:22:40,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/2c99101b73bf433a990119f4de3c8d99 2024-11-08T02:22:40,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/a1562ac02c6e4041b6246e9cbf743d7e is 50, key is test_row_0/B:col10/1731032560699/Put/seqid=0 2024-11-08T02:22:40,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741874_1050 (size=12151) 2024-11-08T02:22:40,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032620828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032620838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032620839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:40,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032621038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032621049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032621049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T02:22:41,209 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:22:41,213 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:22:41,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-08T02:22:41,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/a1562ac02c6e4041b6246e9cbf743d7e 2024-11-08T02:22:41,216 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:22:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T02:22:41,218 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:22:41,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:22:41,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/263be9a44c7649ec95dc0a717ff26c98 is 50, key is test_row_0/C:col10/1731032560699/Put/seqid=0 2024-11-08T02:22:41,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032621238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032621238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741875_1051 (size=12151) 2024-11-08T02:22:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/263be9a44c7649ec95dc0a717ff26c98 2024-11-08T02:22:41,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/2c99101b73bf433a990119f4de3c8d99 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2c99101b73bf433a990119f4de3c8d99 2024-11-08T02:22:41,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2c99101b73bf433a990119f4de3c8d99, entries=200, sequenceid=202, filesize=14.2 K 2024-11-08T02:22:41,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/a1562ac02c6e4041b6246e9cbf743d7e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a1562ac02c6e4041b6246e9cbf743d7e 2024-11-08T02:22:41,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a1562ac02c6e4041b6246e9cbf743d7e, entries=150, sequenceid=202, filesize=11.9 K 2024-11-08T02:22:41,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/263be9a44c7649ec95dc0a717ff26c98 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/263be9a44c7649ec95dc0a717ff26c98 2024-11-08T02:22:41,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/263be9a44c7649ec95dc0a717ff26c98, entries=150, sequenceid=202, filesize=11.9 K 2024-11-08T02:22:41,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8469534ae513a00a18db97d665b015cd in 601ms, sequenceid=202, compaction requested=true 2024-11-08T02:22:41,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:41,303 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:22:41,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:41,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:41,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:41,304 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:22:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:41,306 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53692 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:22:41,306 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:41,306 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,306 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e89b2708699d4f108082bd36737cb678, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/332f2b1131be4c318aa7f0bc354bd3bf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/57661a2e566d49d48ecc52e9ecb339ff, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2c99101b73bf433a990119f4de3c8d99] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=52.4 K 2024-11-08T02:22:41,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:41,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:41,307 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting e89b2708699d4f108082bd36737cb678, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731032558050 2024-11-08T02:22:41,307 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:22:41,308 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 332f2b1131be4c318aa7f0bc354bd3bf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731032558468 2024-11-08T02:22:41,308 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:41,308 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,308 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/0b02ed2c55df4909a26aae7bffe59dfa, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/718201a4bd104070b5ab690fb0792cd0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e15195eb7b4f4580b25ad3509013356a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a1562ac02c6e4041b6246e9cbf743d7e] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=47.8 K 2024-11-08T02:22:41,309 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57661a2e566d49d48ecc52e9ecb339ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731032558816 2024-11-08T02:22:41,309 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b02ed2c55df4909a26aae7bffe59dfa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731032558050 2024-11-08T02:22:41,310 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c99101b73bf433a990119f4de3c8d99, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731032560061 2024-11-08T02:22:41,310 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 718201a4bd104070b5ab690fb0792cd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731032558471 2024-11-08T02:22:41,311 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting e15195eb7b4f4580b25ad3509013356a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731032558816 2024-11-08T02:22:41,312 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a1562ac02c6e4041b6246e9cbf743d7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731032560061 2024-11-08T02:22:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T02:22:41,334 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:41,335 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/1bb84ee74b95426cb451c63037d44dd0 is 50, key is test_row_0/A:col10/1731032560699/Put/seqid=0 2024-11-08T02:22:41,345 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:41,345 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/92d78fac33b847a0876c59f004bc72cd is 50, key is test_row_0/B:col10/1731032560699/Put/seqid=0 2024-11-08T02:22:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:41,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:41,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:22:41,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741876_1052 (size=12595) 2024-11-08T02:22:41,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741877_1053 (size=12595) 2024-11-08T02:22:41,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:41,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:41,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,380 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/1bb84ee74b95426cb451c63037d44dd0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1bb84ee74b95426cb451c63037d44dd0 2024-11-08T02:22:41,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/8b4c0d73b89c438783e9b749e3245fad is 50, key is test_row_0/A:col10/1731032560705/Put/seqid=0 2024-11-08T02:22:41,392 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/92d78fac33b847a0876c59f004bc72cd as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/92d78fac33b847a0876c59f004bc72cd 2024-11-08T02:22:41,407 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into 1bb84ee74b95426cb451c63037d44dd0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:41,407 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into 92d78fac33b847a0876c59f004bc72cd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:41,407 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:41,407 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:41,407 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=12, startTime=1731032561303; duration=0sec 2024-11-08T02:22:41,407 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=12, startTime=1731032561304; duration=0sec 2024-11-08T02:22:41,407 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:41,408 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:41,408 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:41,408 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:41,408 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:22:41,413 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:22:41,413 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:41,414 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,414 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/e9df5b925325499bb8c145f3b6f9a96b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/86e4eaed86664f84a19611cddaf82246, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/20a3ae5a19464281b9a47c40f38f01f8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/263be9a44c7649ec95dc0a717ff26c98] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=47.8 K 2024-11-08T02:22:41,415 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9df5b925325499bb8c145f3b6f9a96b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731032558050 2024-11-08T02:22:41,415 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86e4eaed86664f84a19611cddaf82246, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731032558471 2024-11-08T02:22:41,417 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20a3ae5a19464281b9a47c40f38f01f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731032558816 2024-11-08T02:22:41,417 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 263be9a44c7649ec95dc0a717ff26c98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731032560061 2024-11-08T02:22:41,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741878_1054 (size=16931) 2024-11-08T02:22:41,443 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:41,444 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/5bbf273b7c024b57b473f5757f593fca is 50, key is test_row_0/C:col10/1731032560699/Put/seqid=0 2024-11-08T02:22:41,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741879_1055 (size=12595) 2024-11-08T02:22:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032621442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032621452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032621457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T02:22:41,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032621558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032621568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032621569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032621768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032621778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:41,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032621778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:41,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:41,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:41,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/8b4c0d73b89c438783e9b749e3245fad 2024-11-08T02:22:41,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T02:22:41,863 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/5bbf273b7c024b57b473f5757f593fca as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/5bbf273b7c024b57b473f5757f593fca 2024-11-08T02:22:41,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/73165eb203af4b8384a4df451c34f104 is 50, key is test_row_0/B:col10/1731032560705/Put/seqid=0 2024-11-08T02:22:41,872 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into 5bbf273b7c024b57b473f5757f593fca(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:41,872 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:41,872 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=12, startTime=1731032561305; duration=0sec 2024-11-08T02:22:41,873 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:41,873 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:41,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741880_1056 (size=12151) 2024-11-08T02:22:41,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/73165eb203af4b8384a4df451c34f104 2024-11-08T02:22:41,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/ee20f4905e9849a19c0a5b3fd88a95df is 50, key is test_row_0/C:col10/1731032560705/Put/seqid=0 2024-11-08T02:22:41,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741881_1057 (size=12151) 2024-11-08T02:22:41,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/ee20f4905e9849a19c0a5b3fd88a95df 2024-11-08T02:22:41,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/8b4c0d73b89c438783e9b749e3245fad as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/8b4c0d73b89c438783e9b749e3245fad 2024-11-08T02:22:41,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/8b4c0d73b89c438783e9b749e3245fad, entries=250, sequenceid=214, filesize=16.5 K 2024-11-08T02:22:41,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/73165eb203af4b8384a4df451c34f104 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/73165eb203af4b8384a4df451c34f104 2024-11-08T02:22:41,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/73165eb203af4b8384a4df451c34f104, entries=150, sequenceid=214, filesize=11.9 K 2024-11-08T02:22:41,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:41,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/ee20f4905e9849a19c0a5b3fd88a95df as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/ee20f4905e9849a19c0a5b3fd88a95df 2024-11-08T02:22:41,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:41,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:41,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/ee20f4905e9849a19c0a5b3fd88a95df, entries=150, sequenceid=214, filesize=11.9 K 2024-11-08T02:22:42,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 8469534ae513a00a18db97d665b015cd in 646ms, sequenceid=214, compaction requested=false 2024-11-08T02:22:42,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:42,094 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:42,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:22:42,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/adea0ce45c464738ae3715ee46968d25 is 50, key is test_row_0/A:col10/1731032561446/Put/seqid=0 2024-11-08T02:22:42,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032622114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032622115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032622120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:42,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:42,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741882_1058 (size=14541) 2024-11-08T02:22:42,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032622229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032622229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032622239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032622258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,266 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 8469534ae513a00a18db97d665b015cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:22:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032622269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,271 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 8469534ae513a00a18db97d665b015cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:22:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:42,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:42,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T02:22:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032622448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032622449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032622449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:42,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:42,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/adea0ce45c464738ae3715ee46968d25 2024-11-08T02:22:42,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/503d9a0ad98f417e81e374eec1c40137 is 50, key is test_row_0/B:col10/1731032561446/Put/seqid=0 2024-11-08T02:22:42,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:42,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:42,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741883_1059 (size=12151) 2024-11-08T02:22:42,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032622758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032622758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:42,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032622758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:42,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:42,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:42,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:43,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/503d9a0ad98f417e81e374eec1c40137 2024-11-08T02:22:43,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/3e3fe53113f34a759ec92812ce6889f6 is 50, key is test_row_0/C:col10/1731032561446/Put/seqid=0 2024-11-08T02:22:43,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:43,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:43,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:43,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741884_1060 (size=12151) 2024-11-08T02:22:43,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/3e3fe53113f34a759ec92812ce6889f6 2024-11-08T02:22:43,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/adea0ce45c464738ae3715ee46968d25 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/adea0ce45c464738ae3715ee46968d25 2024-11-08T02:22:43,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/adea0ce45c464738ae3715ee46968d25, entries=200, sequenceid=244, filesize=14.2 K 2024-11-08T02:22:43,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/503d9a0ad98f417e81e374eec1c40137 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/503d9a0ad98f417e81e374eec1c40137 2024-11-08T02:22:43,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/503d9a0ad98f417e81e374eec1c40137, entries=150, sequenceid=244, filesize=11.9 K 2024-11-08T02:22:43,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/3e3fe53113f34a759ec92812ce6889f6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/3e3fe53113f34a759ec92812ce6889f6 2024-11-08T02:22:43,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/3e3fe53113f34a759ec92812ce6889f6, entries=150, sequenceid=244, filesize=11.9 K 2024-11-08T02:22:43,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 8469534ae513a00a18db97d665b015cd in 1104ms, sequenceid=244, compaction requested=true 2024-11-08T02:22:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:43,198 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:43,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:43,199 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:43,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:43,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:43,200 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44067 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:43,201 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:43,201 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:43,201 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1bb84ee74b95426cb451c63037d44dd0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/8b4c0d73b89c438783e9b749e3245fad, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/adea0ce45c464738ae3715ee46968d25] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=43.0 K 2024-11-08T02:22:43,202 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:43,202 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:43,202 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bb84ee74b95426cb451c63037d44dd0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731032560061 2024-11-08T02:22:43,202 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:43,202 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/92d78fac33b847a0876c59f004bc72cd, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/73165eb203af4b8384a4df451c34f104, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/503d9a0ad98f417e81e374eec1c40137] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=36.0 K 2024-11-08T02:22:43,203 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b4c0d73b89c438783e9b749e3245fad, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731032560705 2024-11-08T02:22:43,203 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 92d78fac33b847a0876c59f004bc72cd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731032560061 2024-11-08T02:22:43,203 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting adea0ce45c464738ae3715ee46968d25, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731032561437 2024-11-08T02:22:43,204 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 73165eb203af4b8384a4df451c34f104, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731032560705 2024-11-08T02:22:43,204 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 503d9a0ad98f417e81e374eec1c40137, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731032561446 2024-11-08T02:22:43,232 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:43,233 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/4e29c301486f4473a63d8aae55f6b65b is 50, key is test_row_0/B:col10/1731032561446/Put/seqid=0 2024-11-08T02:22:43,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T02:22:43,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:43,247 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-08T02:22:43,249 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:43,250 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/caa8d8b9ba504c3899bb9aedf890ec24 is 50, key is test_row_0/A:col10/1731032561446/Put/seqid=0 2024-11-08T02:22:43,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/04adfd40ced54a6aa1649d546e0525eb is 50, key is test_row_0/A:col10/1731032562115/Put/seqid=0 2024-11-08T02:22:43,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:43,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741885_1061 (size=12697) 2024-11-08T02:22:43,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741886_1062 (size=12697) 2024-11-08T02:22:43,312 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/4e29c301486f4473a63d8aae55f6b65b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4e29c301486f4473a63d8aae55f6b65b 2024-11-08T02:22:43,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741887_1063 (size=12151) 2024-11-08T02:22:43,327 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/caa8d8b9ba504c3899bb9aedf890ec24 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/caa8d8b9ba504c3899bb9aedf890ec24 2024-11-08T02:22:43,329 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/04adfd40ced54a6aa1649d546e0525eb 2024-11-08T02:22:43,341 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into caa8d8b9ba504c3899bb9aedf890ec24(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:43,341 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:43,341 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=13, startTime=1731032563198; duration=0sec 2024-11-08T02:22:43,341 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:43,341 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:43,341 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:43,345 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:43,345 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:43,345 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into 4e29c301486f4473a63d8aae55f6b65b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:43,345 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:43,345 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:43,345 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=13, startTime=1731032563198; duration=0sec 2024-11-08T02:22:43,345 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/5bbf273b7c024b57b473f5757f593fca, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/ee20f4905e9849a19c0a5b3fd88a95df, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/3e3fe53113f34a759ec92812ce6889f6] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=36.0 K 2024-11-08T02:22:43,345 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:43,345 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:43,346 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bbf273b7c024b57b473f5757f593fca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731032560061 2024-11-08T02:22:43,347 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee20f4905e9849a19c0a5b3fd88a95df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731032560705 2024-11-08T02:22:43,348 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e3fe53113f34a759ec92812ce6889f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731032561446 2024-11-08T02:22:43,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/4d84ad0bb7a849c3bcb17aa41038214a is 50, key is test_row_0/B:col10/1731032562115/Put/seqid=0 2024-11-08T02:22:43,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032623345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032623355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032623360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T02:22:43,373 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:43,375 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/596fc7a345ef4b2ba5d33d1977f63b4e is 50, key is test_row_0/C:col10/1731032561446/Put/seqid=0 2024-11-08T02:22:43,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741888_1064 (size=12151) 2024-11-08T02:22:43,390 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/4d84ad0bb7a849c3bcb17aa41038214a 2024-11-08T02:22:43,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/7e353b14a14b4f2e8a3f83b9135aa9ef is 50, key is test_row_0/C:col10/1731032562115/Put/seqid=0 2024-11-08T02:22:43,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741889_1065 (size=12697) 2024-11-08T02:22:43,423 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/596fc7a345ef4b2ba5d33d1977f63b4e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/596fc7a345ef4b2ba5d33d1977f63b4e 2024-11-08T02:22:43,437 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into 596fc7a345ef4b2ba5d33d1977f63b4e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:43,437 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:43,437 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=13, startTime=1731032563199; duration=0sec 2024-11-08T02:22:43,437 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:43,437 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:43,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741890_1066 (size=12151) 2024-11-08T02:22:43,447 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/7e353b14a14b4f2e8a3f83b9135aa9ef 2024-11-08T02:22:43,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/04adfd40ced54a6aa1649d546e0525eb as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/04adfd40ced54a6aa1649d546e0525eb 2024-11-08T02:22:43,462 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/04adfd40ced54a6aa1649d546e0525eb, entries=150, sequenceid=253, filesize=11.9 K 2024-11-08T02:22:43,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/4d84ad0bb7a849c3bcb17aa41038214a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4d84ad0bb7a849c3bcb17aa41038214a 2024-11-08T02:22:43,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032623470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032623470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032623470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,478 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4d84ad0bb7a849c3bcb17aa41038214a, entries=150, sequenceid=253, filesize=11.9 K 2024-11-08T02:22:43,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/7e353b14a14b4f2e8a3f83b9135aa9ef as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/7e353b14a14b4f2e8a3f83b9135aa9ef 2024-11-08T02:22:43,499 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/7e353b14a14b4f2e8a3f83b9135aa9ef, entries=150, sequenceid=253, filesize=11.9 K 2024-11-08T02:22:43,500 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 8469534ae513a00a18db97d665b015cd in 253ms, sequenceid=253, compaction requested=false 2024-11-08T02:22:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-08T02:22:43,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-08T02:22:43,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-08T02:22:43,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2860 sec 2024-11-08T02:22:43,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.2970 sec 2024-11-08T02:22:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:43,681 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:43,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T02:22:43,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032623685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032623688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032623688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/fd9a9304f3634d46afc243c74f368d18 is 50, key is test_row_0/A:col10/1731032563680/Put/seqid=0 2024-11-08T02:22:43,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741891_1067 (size=14741) 2024-11-08T02:22:43,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/fd9a9304f3634d46afc243c74f368d18 2024-11-08T02:22:43,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/093df2fd5ece4a6694df9c3471bc2407 is 50, key is test_row_0/B:col10/1731032563680/Put/seqid=0 2024-11-08T02:22:43,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032623798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032623799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032623800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:43,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741892_1068 (size=12301) 2024-11-08T02:22:43,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/093df2fd5ece4a6694df9c3471bc2407 2024-11-08T02:22:43,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/a2e9d0bc245c44209bc0933d8d93c5bd is 50, key is test_row_0/C:col10/1731032563680/Put/seqid=0 2024-11-08T02:22:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741893_1069 (size=12301) 2024-11-08T02:22:44,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032624008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032624011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032624013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/a2e9d0bc245c44209bc0933d8d93c5bd 2024-11-08T02:22:44,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/fd9a9304f3634d46afc243c74f368d18 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fd9a9304f3634d46afc243c74f368d18 2024-11-08T02:22:44,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fd9a9304f3634d46afc243c74f368d18, entries=200, sequenceid=286, filesize=14.4 K 2024-11-08T02:22:44,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/093df2fd5ece4a6694df9c3471bc2407 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/093df2fd5ece4a6694df9c3471bc2407 2024-11-08T02:22:44,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/093df2fd5ece4a6694df9c3471bc2407, entries=150, sequenceid=286, filesize=12.0 K 2024-11-08T02:22:44,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/a2e9d0bc245c44209bc0933d8d93c5bd as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a2e9d0bc245c44209bc0933d8d93c5bd 2024-11-08T02:22:44,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032624289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,294 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 8469534ae513a00a18db97d665b015cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:22:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032624289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,296 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 8469534ae513a00a18db97d665b015cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:22:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a2e9d0bc245c44209bc0933d8d93c5bd, entries=150, sequenceid=286, filesize=12.0 K 2024-11-08T02:22:44,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for 8469534ae513a00a18db97d665b015cd in 625ms, sequenceid=286, compaction requested=true 2024-11-08T02:22:44,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:44,307 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:44,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:44,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:44,307 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:44,309 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:44,309 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:44,309 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:44,309 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4e29c301486f4473a63d8aae55f6b65b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4d84ad0bb7a849c3bcb17aa41038214a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/093df2fd5ece4a6694df9c3471bc2407] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=36.3 K 2024-11-08T02:22:44,309 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:44,309 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:44,309 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:44,310 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/caa8d8b9ba504c3899bb9aedf890ec24, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/04adfd40ced54a6aa1649d546e0525eb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fd9a9304f3634d46afc243c74f368d18] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=38.7 K 2024-11-08T02:22:44,310 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e29c301486f4473a63d8aae55f6b65b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731032561446 2024-11-08T02:22:44,311 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting caa8d8b9ba504c3899bb9aedf890ec24, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731032561446 2024-11-08T02:22:44,312 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04adfd40ced54a6aa1649d546e0525eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731032562108 2024-11-08T02:22:44,312 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d84ad0bb7a849c3bcb17aa41038214a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731032562108 2024-11-08T02:22:44,312 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 093df2fd5ece4a6694df9c3471bc2407, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731032563357 2024-11-08T02:22:44,312 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd9a9304f3634d46afc243c74f368d18, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731032563357 2024-11-08T02:22:44,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:44,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:44,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:44,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:44,328 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:44,328 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/633bd126d2704c8fa7630b476a6c930a is 50, key is test_row_0/A:col10/1731032563680/Put/seqid=0 2024-11-08T02:22:44,336 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:44,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:22:44,337 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:44,338 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/7feb6f150748491697fe149e00816733 is 50, key is test_row_0/B:col10/1731032563680/Put/seqid=0 2024-11-08T02:22:44,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/11e3c0a23c85403b8abd03f3f993f089 is 50, key is test_row_0/A:col10/1731032564328/Put/seqid=0 2024-11-08T02:22:44,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741894_1070 (size=12949) 2024-11-08T02:22:44,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741895_1071 (size=12949) 2024-11-08T02:22:44,380 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/633bd126d2704c8fa7630b476a6c930a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/633bd126d2704c8fa7630b476a6c930a 2024-11-08T02:22:44,388 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/7feb6f150748491697fe149e00816733 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/7feb6f150748491697fe149e00816733 2024-11-08T02:22:44,398 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into 633bd126d2704c8fa7630b476a6c930a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:44,398 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:44,399 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=13, startTime=1731032564307; duration=0sec 2024-11-08T02:22:44,399 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:44,399 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:44,399 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:44,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741896_1072 (size=14741) 2024-11-08T02:22:44,402 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into 7feb6f150748491697fe149e00816733(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:44,402 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:44,402 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=13, startTime=1731032564307; duration=0sec 2024-11-08T02:22:44,403 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:44,403 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:44,404 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:44,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/11e3c0a23c85403b8abd03f3f993f089 2024-11-08T02:22:44,405 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:44,405 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:44,407 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/596fc7a345ef4b2ba5d33d1977f63b4e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/7e353b14a14b4f2e8a3f83b9135aa9ef, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a2e9d0bc245c44209bc0933d8d93c5bd] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=36.3 K 2024-11-08T02:22:44,408 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 596fc7a345ef4b2ba5d33d1977f63b4e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731032561446 2024-11-08T02:22:44,410 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e353b14a14b4f2e8a3f83b9135aa9ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731032562108 2024-11-08T02:22:44,411 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2e9d0bc245c44209bc0933d8d93c5bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731032563357 2024-11-08T02:22:44,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/ec38824da9c94930b67d515e29b2abde is 50, key is test_row_0/B:col10/1731032564328/Put/seqid=0 2024-11-08T02:22:44,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032624426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,437 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:44,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032624431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,438 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/9d5340cc08c44d71b251eb2d4a23ddcb is 50, key is test_row_0/C:col10/1731032563680/Put/seqid=0 2024-11-08T02:22:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032624433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741897_1073 (size=12301) 2024-11-08T02:22:44,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/ec38824da9c94930b67d515e29b2abde 2024-11-08T02:22:44,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/cbc2fd3ac04840e1a0a02f95645670b9 is 50, key is test_row_0/C:col10/1731032564328/Put/seqid=0 2024-11-08T02:22:44,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741898_1074 (size=12949) 2024-11-08T02:22:44,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741899_1075 (size=12301) 2024-11-08T02:22:44,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/cbc2fd3ac04840e1a0a02f95645670b9 2024-11-08T02:22:44,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/11e3c0a23c85403b8abd03f3f993f089 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/11e3c0a23c85403b8abd03f3f993f089 2024-11-08T02:22:44,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/11e3c0a23c85403b8abd03f3f993f089, entries=200, sequenceid=297, filesize=14.4 K 2024-11-08T02:22:44,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/ec38824da9c94930b67d515e29b2abde as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec38824da9c94930b67d515e29b2abde 2024-11-08T02:22:44,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec38824da9c94930b67d515e29b2abde, entries=150, sequenceid=297, filesize=12.0 K 2024-11-08T02:22:44,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/cbc2fd3ac04840e1a0a02f95645670b9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/cbc2fd3ac04840e1a0a02f95645670b9 2024-11-08T02:22:44,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032624539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/cbc2fd3ac04840e1a0a02f95645670b9, entries=150, sequenceid=297, filesize=12.0 K 2024-11-08T02:22:44,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8469534ae513a00a18db97d665b015cd in 212ms, sequenceid=297, compaction requested=false 2024-11-08T02:22:44,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:44,553 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:44,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T02:22:44,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/85b043222c3f47ebb15d600f1f303673 is 50, key is test_row_0/A:col10/1731032564551/Put/seqid=0 2024-11-08T02:22:44,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032624578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032624580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741900_1076 (size=17181) 2024-11-08T02:22:44,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032624688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032624688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032624748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,886 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/9d5340cc08c44d71b251eb2d4a23ddcb as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/9d5340cc08c44d71b251eb2d4a23ddcb 2024-11-08T02:22:44,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032624898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032624898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:44,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:44,902 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into 9d5340cc08c44d71b251eb2d4a23ddcb(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:44,902 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:44,902 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=13, startTime=1731032564323; duration=0sec 2024-11-08T02:22:44,902 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:44,902 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:45,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/85b043222c3f47ebb15d600f1f303673 2024-11-08T02:22:45,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/a32016a868f24dc4b13c5e6d751360b0 is 50, key is test_row_0/B:col10/1731032564551/Put/seqid=0 2024-11-08T02:22:45,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741901_1077 (size=12301) 2024-11-08T02:22:45,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032625058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:45,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:45,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032625208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:45,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:45,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032625208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T02:22:45,378 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:22:45,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:22:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-08T02:22:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T02:22:45,380 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:22:45,381 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:22:45,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:22:45,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/a32016a868f24dc4b13c5e6d751360b0 2024-11-08T02:22:45,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/b7fab156b04a406b8e0763819534155b is 50, key is test_row_0/C:col10/1731032564551/Put/seqid=0 2024-11-08T02:22:45,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741902_1078 (size=12301) 2024-11-08T02:22:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T02:22:45,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T02:22:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:45,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:45,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:45,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032625568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:45,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T02:22:45,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:45,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:45,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:45,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:45,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T02:22:45,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032625719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:45,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032625719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:45,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T02:22:45,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:45,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:45,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:45,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:45,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:45,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/b7fab156b04a406b8e0763819534155b 2024-11-08T02:22:45,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/85b043222c3f47ebb15d600f1f303673 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/85b043222c3f47ebb15d600f1f303673 2024-11-08T02:22:45,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/85b043222c3f47ebb15d600f1f303673, entries=250, sequenceid=326, filesize=16.8 K 2024-11-08T02:22:45,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/a32016a868f24dc4b13c5e6d751360b0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a32016a868f24dc4b13c5e6d751360b0 2024-11-08T02:22:45,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a32016a868f24dc4b13c5e6d751360b0, entries=150, sequenceid=326, filesize=12.0 K 2024-11-08T02:22:45,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/b7fab156b04a406b8e0763819534155b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b7fab156b04a406b8e0763819534155b 2024-11-08T02:22:45,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b7fab156b04a406b8e0763819534155b, entries=150, sequenceid=326, filesize=12.0 K 2024-11-08T02:22:45,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 8469534ae513a00a18db97d665b015cd in 1354ms, sequenceid=326, compaction requested=true 2024-11-08T02:22:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:45,909 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:45,909 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:45,910 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44871 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:45,910 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:45,910 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:45,910 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:45,910 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:45,910 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:45,911 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/633bd126d2704c8fa7630b476a6c930a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/11e3c0a23c85403b8abd03f3f993f089, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/85b043222c3f47ebb15d600f1f303673] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=43.8 K 2024-11-08T02:22:45,911 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/7feb6f150748491697fe149e00816733, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec38824da9c94930b67d515e29b2abde, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a32016a868f24dc4b13c5e6d751360b0] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=36.7 K 2024-11-08T02:22:45,911 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 7feb6f150748491697fe149e00816733, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731032563357 2024-11-08T02:22:45,911 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 633bd126d2704c8fa7630b476a6c930a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731032563357 2024-11-08T02:22:45,912 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11e3c0a23c85403b8abd03f3f993f089, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731032563685 2024-11-08T02:22:45,912 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting ec38824da9c94930b67d515e29b2abde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731032563685 2024-11-08T02:22:45,912 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85b043222c3f47ebb15d600f1f303673, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731032564414 2024-11-08T02:22:45,912 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a32016a868f24dc4b13c5e6d751360b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731032564423 2024-11-08T02:22:45,926 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:45,926 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/1999f6235f6e45f59b0f03cf792daabb is 50, key is test_row_0/A:col10/1731032564551/Put/seqid=0 2024-11-08T02:22:45,929 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#66 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:45,930 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/999d2b88e65c4fa89301518bfd351b0d is 50, key is test_row_0/B:col10/1731032564551/Put/seqid=0 2024-11-08T02:22:45,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741903_1079 (size=13051) 2024-11-08T02:22:45,950 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/1999f6235f6e45f59b0f03cf792daabb as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1999f6235f6e45f59b0f03cf792daabb 2024-11-08T02:22:45,958 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into 1999f6235f6e45f59b0f03cf792daabb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:45,958 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:45,958 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=13, startTime=1731032565909; duration=0sec 2024-11-08T02:22:45,958 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:45,958 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:45,958 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:45,961 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:45,962 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:45,962 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:45,962 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/9d5340cc08c44d71b251eb2d4a23ddcb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/cbc2fd3ac04840e1a0a02f95645670b9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b7fab156b04a406b8e0763819534155b] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=36.7 K 2024-11-08T02:22:45,962 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d5340cc08c44d71b251eb2d4a23ddcb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731032563357 2024-11-08T02:22:45,963 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc2fd3ac04840e1a0a02f95645670b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731032563685 2024-11-08T02:22:45,963 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7fab156b04a406b8e0763819534155b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731032564423 2024-11-08T02:22:45,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741904_1080 (size=13051) 2024-11-08T02:22:45,976 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:45,976 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/34c130b0f1534dae8ba46a7345cf5e5c is 50, key is test_row_0/C:col10/1731032564551/Put/seqid=0 2024-11-08T02:22:45,983 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/999d2b88e65c4fa89301518bfd351b0d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/999d2b88e65c4fa89301518bfd351b0d 2024-11-08T02:22:45,995 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into 999d2b88e65c4fa89301518bfd351b0d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:45,995 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:45,995 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=13, startTime=1731032565909; duration=0sec 2024-11-08T02:22:45,995 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:45,995 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:46,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T02:22:46,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:46,002 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-08T02:22:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T02:22:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/6b8eb9202c754176b00e66a25301f0c9 is 50, key is test_row_0/A:col10/1731032564562/Put/seqid=0 2024-11-08T02:22:46,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741905_1081 (size=13051) 2024-11-08T02:22:46,036 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/34c130b0f1534dae8ba46a7345cf5e5c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/34c130b0f1534dae8ba46a7345cf5e5c 2024-11-08T02:22:46,047 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into 34c130b0f1534dae8ba46a7345cf5e5c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:46,047 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:46,047 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=13, startTime=1731032565909; duration=0sec 2024-11-08T02:22:46,048 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:46,048 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:46,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741906_1082 (size=12301) 2024-11-08T02:22:46,450 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/6b8eb9202c754176b00e66a25301f0c9 2024-11-08T02:22:46,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/24b1f3ee55dc46459f6c4b184edbe03c is 50, key is test_row_0/B:col10/1731032564562/Put/seqid=0 2024-11-08T02:22:46,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741907_1083 (size=12301) 2024-11-08T02:22:46,484 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/24b1f3ee55dc46459f6c4b184edbe03c 2024-11-08T02:22:46,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/adef1a219d004c4896957c17fea01c4e is 50, key is test_row_0/C:col10/1731032564562/Put/seqid=0 2024-11-08T02:22:46,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741908_1084 (size=12301) 2024-11-08T02:22:46,505 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/adef1a219d004c4896957c17fea01c4e 2024-11-08T02:22:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T02:22:46,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/6b8eb9202c754176b00e66a25301f0c9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/6b8eb9202c754176b00e66a25301f0c9 2024-11-08T02:22:46,526 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/6b8eb9202c754176b00e66a25301f0c9, entries=150, sequenceid=338, filesize=12.0 K 2024-11-08T02:22:46,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/24b1f3ee55dc46459f6c4b184edbe03c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/24b1f3ee55dc46459f6c4b184edbe03c 2024-11-08T02:22:46,537 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/24b1f3ee55dc46459f6c4b184edbe03c, entries=150, sequenceid=338, filesize=12.0 K 2024-11-08T02:22:46,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/adef1a219d004c4896957c17fea01c4e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/adef1a219d004c4896957c17fea01c4e 2024-11-08T02:22:46,547 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/adef1a219d004c4896957c17fea01c4e, entries=150, sequenceid=338, filesize=12.0 K 2024-11-08T02:22:46,548 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 8469534ae513a00a18db97d665b015cd in 546ms, sequenceid=338, compaction requested=false 2024-11-08T02:22:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-08T02:22:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-08T02:22:46,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-08T02:22:46,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1710 sec 2024-11-08T02:22:46,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1840 sec 2024-11-08T02:22:46,621 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:46,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:22:46,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/2a7343ad4a084c4f9117c37745888328 is 50, key is test_row_0/A:col10/1731032566616/Put/seqid=0 2024-11-08T02:22:46,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741909_1085 (size=12301) 2024-11-08T02:22:46,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/2a7343ad4a084c4f9117c37745888328 2024-11-08T02:22:46,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/dca815b2b8764746bdac1c7ad70173a4 is 50, key is test_row_0/B:col10/1731032566616/Put/seqid=0 2024-11-08T02:22:46,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741910_1086 (size=12301) 2024-11-08T02:22:46,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/dca815b2b8764746bdac1c7ad70173a4 2024-11-08T02:22:46,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/dfe569f3b01d4738a2e2c235ac2a8afe is 50, key is test_row_0/C:col10/1731032566616/Put/seqid=0 2024-11-08T02:22:46,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741911_1087 (size=12301) 2024-11-08T02:22:46,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/dfe569f3b01d4738a2e2c235ac2a8afe 2024-11-08T02:22:46,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032626738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:46,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032626741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:46,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032626742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:46,746 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 8469534ae513a00a18db97d665b015cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:22:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:46,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:46,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/2a7343ad4a084c4f9117c37745888328 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2a7343ad4a084c4f9117c37745888328 2024-11-08T02:22:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:46,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2a7343ad4a084c4f9117c37745888328, entries=150, sequenceid=350, filesize=12.0 K 2024-11-08T02:22:46,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/dca815b2b8764746bdac1c7ad70173a4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/dca815b2b8764746bdac1c7ad70173a4 2024-11-08T02:22:46,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/dca815b2b8764746bdac1c7ad70173a4, entries=150, sequenceid=350, filesize=12.0 K 2024-11-08T02:22:46,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/dfe569f3b01d4738a2e2c235ac2a8afe as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/dfe569f3b01d4738a2e2c235ac2a8afe 2024-11-08T02:22:46,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/dfe569f3b01d4738a2e2c235ac2a8afe, entries=150, sequenceid=350, filesize=12.0 K 2024-11-08T02:22:46,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 8469534ae513a00a18db97d665b015cd in 164ms, sequenceid=350, compaction requested=true 2024-11-08T02:22:46,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:46,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:46,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:46,785 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:46,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:46,785 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:46,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:46,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:46,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:46,787 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:46,787 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:46,787 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:46,787 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/999d2b88e65c4fa89301518bfd351b0d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/24b1f3ee55dc46459f6c4b184edbe03c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/dca815b2b8764746bdac1c7ad70173a4] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=36.8 K 2024-11-08T02:22:46,787 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:46,787 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:46,787 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:46,787 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1999f6235f6e45f59b0f03cf792daabb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/6b8eb9202c754176b00e66a25301f0c9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2a7343ad4a084c4f9117c37745888328] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=36.8 K 2024-11-08T02:22:46,788 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 999d2b88e65c4fa89301518bfd351b0d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731032564423 2024-11-08T02:22:46,788 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1999f6235f6e45f59b0f03cf792daabb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731032564423 2024-11-08T02:22:46,788 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 24b1f3ee55dc46459f6c4b184edbe03c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731032564562 2024-11-08T02:22:46,788 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b8eb9202c754176b00e66a25301f0c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731032564562 2024-11-08T02:22:46,788 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting dca815b2b8764746bdac1c7ad70173a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731032566612 2024-11-08T02:22:46,790 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a7343ad4a084c4f9117c37745888328, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731032566612 2024-11-08T02:22:46,799 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:46,799 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/7dac066ea7744c279819070f68d41ff8 is 50, key is test_row_0/B:col10/1731032566616/Put/seqid=0 2024-11-08T02:22:46,801 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:46,802 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/ad7e55a76da54a7b9e67e69c17b7176c is 50, key is test_row_0/A:col10/1731032566616/Put/seqid=0 2024-11-08T02:22:46,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741912_1088 (size=13153) 2024-11-08T02:22:46,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741913_1089 (size=13153) 2024-11-08T02:22:46,864 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/ad7e55a76da54a7b9e67e69c17b7176c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/ad7e55a76da54a7b9e67e69c17b7176c 2024-11-08T02:22:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:46,865 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:46,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T02:22:46,877 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into ad7e55a76da54a7b9e67e69c17b7176c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:46,877 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:46,877 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=13, startTime=1731032566785; duration=0sec 2024-11-08T02:22:46,877 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:46,877 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:46,877 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:46,879 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:46,880 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:46,880 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:46,880 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/34c130b0f1534dae8ba46a7345cf5e5c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/adef1a219d004c4896957c17fea01c4e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/dfe569f3b01d4738a2e2c235ac2a8afe] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=36.8 K 2024-11-08T02:22:46,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/3b77938ab0824d9296ef77b8af8c9f14 is 50, key is test_row_0/A:col10/1731032566864/Put/seqid=0 2024-11-08T02:22:46,888 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34c130b0f1534dae8ba46a7345cf5e5c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731032564423 2024-11-08T02:22:46,889 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting adef1a219d004c4896957c17fea01c4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731032564562 2024-11-08T02:22:46,891 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfe569f3b01d4738a2e2c235ac2a8afe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731032566612 2024-11-08T02:22:46,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032626888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:46,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032626892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:46,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741914_1090 (size=14741) 2024-11-08T02:22:46,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/3b77938ab0824d9296ef77b8af8c9f14 2024-11-08T02:22:46,942 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:46,943 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/0dcaddcf39c04ec89572145a47f7e555 is 50, key is test_row_0/C:col10/1731032566616/Put/seqid=0 2024-11-08T02:22:46,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/11323b1b82184811b0a30e9c23f748cc is 50, key is test_row_0/B:col10/1731032566864/Put/seqid=0 2024-11-08T02:22:46,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741915_1091 (size=13153) 2024-11-08T02:22:46,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741916_1092 (size=12301) 2024-11-08T02:22:47,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032626998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:47,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032627007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:47,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032627208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:47,217 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/7dac066ea7744c279819070f68d41ff8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/7dac066ea7744c279819070f68d41ff8 2024-11-08T02:22:47,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032627220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:47,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:47,227 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into 7dac066ea7744c279819070f68d41ff8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:47,227 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:47,227 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=13, startTime=1731032566785; duration=0sec 2024-11-08T02:22:47,228 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:47,228 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:47,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/11323b1b82184811b0a30e9c23f748cc 2024-11-08T02:22:47,379 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/0dcaddcf39c04ec89572145a47f7e555 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/0dcaddcf39c04ec89572145a47f7e555 2024-11-08T02:22:47,388 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into 0dcaddcf39c04ec89572145a47f7e555(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:47,388 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:47,388 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=13, startTime=1731032566785; duration=0sec 2024-11-08T02:22:47,389 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:47,389 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:47,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/d13fb045ad3142e993f3b7b3eb344752 is 50, key is test_row_0/C:col10/1731032566864/Put/seqid=0 2024-11-08T02:22:47,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741917_1093 (size=12301) 2024-11-08T02:22:47,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/d13fb045ad3142e993f3b7b3eb344752 2024-11-08T02:22:47,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/3b77938ab0824d9296ef77b8af8c9f14 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/3b77938ab0824d9296ef77b8af8c9f14 2024-11-08T02:22:47,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/3b77938ab0824d9296ef77b8af8c9f14, entries=200, sequenceid=378, filesize=14.4 K 2024-11-08T02:22:47,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/11323b1b82184811b0a30e9c23f748cc as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/11323b1b82184811b0a30e9c23f748cc 2024-11-08T02:22:47,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/11323b1b82184811b0a30e9c23f748cc, entries=150, sequenceid=378, filesize=12.0 K 2024-11-08T02:22:47,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/d13fb045ad3142e993f3b7b3eb344752 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/d13fb045ad3142e993f3b7b3eb344752 2024-11-08T02:22:47,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/d13fb045ad3142e993f3b7b3eb344752, entries=150, sequenceid=378, filesize=12.0 K 2024-11-08T02:22:47,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 8469534ae513a00a18db97d665b015cd in 565ms, sequenceid=378, compaction requested=false 2024-11-08T02:22:47,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:47,526 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:47,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:22:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T02:22:47,529 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:22:47,529 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:22:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-08T02:22:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T02:22:47,538 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:22:47,539 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:22:47,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:22:47,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/e071280e00cc4414ad9bee76847601f8 is 50, key is test_row_0/A:col10/1731032567520/Put/seqid=0 2024-11-08T02:22:47,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741918_1094 (size=12301) 2024-11-08T02:22:47,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/e071280e00cc4414ad9bee76847601f8 2024-11-08T02:22:47,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/c1b73714d77e4ca1bbd2d7259c985d5b is 50, key is test_row_0/B:col10/1731032567520/Put/seqid=0 2024-11-08T02:22:47,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741919_1095 (size=12301) 2024-11-08T02:22:47,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/c1b73714d77e4ca1bbd2d7259c985d5b 2024-11-08T02:22:47,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/de6694058e2b4026a544752361437593 is 50, key is test_row_0/C:col10/1731032567520/Put/seqid=0 2024-11-08T02:22:47,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032627592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:47,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:47,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741920_1096 (size=12301) 2024-11-08T02:22:47,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032627599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T02:22:47,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T02:22:47,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:47,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:47,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:47,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:47,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:47,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032627708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:47,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032627709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:47,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:47,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T02:22:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:47,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T02:22:47,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032627918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032627918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/de6694058e2b4026a544752361437593 2024-11-08T02:22:48,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T02:22:48,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:48,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:48,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:48,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:48,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:48,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/e071280e00cc4414ad9bee76847601f8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e071280e00cc4414ad9bee76847601f8 2024-11-08T02:22:48,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e071280e00cc4414ad9bee76847601f8, entries=150, sequenceid=392, filesize=12.0 K 2024-11-08T02:22:48,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/c1b73714d77e4ca1bbd2d7259c985d5b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/c1b73714d77e4ca1bbd2d7259c985d5b 2024-11-08T02:22:48,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/c1b73714d77e4ca1bbd2d7259c985d5b, entries=150, sequenceid=392, filesize=12.0 K 2024-11-08T02:22:48,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/de6694058e2b4026a544752361437593 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/de6694058e2b4026a544752361437593 2024-11-08T02:22:48,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/de6694058e2b4026a544752361437593, entries=150, sequenceid=392, filesize=12.0 K 2024-11-08T02:22:48,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8469534ae513a00a18db97d665b015cd in 530ms, sequenceid=392, compaction requested=true 2024-11-08T02:22:48,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:48,057 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:48,057 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:48,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:48,058 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:48,058 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:48,058 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:48,058 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:48,058 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:48,058 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:48,058 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/ad7e55a76da54a7b9e67e69c17b7176c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/3b77938ab0824d9296ef77b8af8c9f14, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e071280e00cc4414ad9bee76847601f8] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=39.3 K 2024-11-08T02:22:48,058 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/7dac066ea7744c279819070f68d41ff8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/11323b1b82184811b0a30e9c23f748cc, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/c1b73714d77e4ca1bbd2d7259c985d5b] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=36.9 K 2024-11-08T02:22:48,059 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dac066ea7744c279819070f68d41ff8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731032566612 2024-11-08T02:22:48,059 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad7e55a76da54a7b9e67e69c17b7176c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731032566612 2024-11-08T02:22:48,061 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 11323b1b82184811b0a30e9c23f748cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731032566738 2024-11-08T02:22:48,062 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b77938ab0824d9296ef77b8af8c9f14, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731032566730 2024-11-08T02:22:48,062 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c1b73714d77e4ca1bbd2d7259c985d5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731032566883 2024-11-08T02:22:48,062 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting e071280e00cc4414ad9bee76847601f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731032566883 2024-11-08T02:22:48,090 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:48,091 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/ea344d37d002489c829b0539dc0457c4 is 50, key is test_row_0/B:col10/1731032567520/Put/seqid=0 2024-11-08T02:22:48,105 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:48,106 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/d20ec8239689472d90191802e9be2a42 is 50, key is test_row_0/A:col10/1731032567520/Put/seqid=0 2024-11-08T02:22:48,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741921_1097 (size=13255) 2024-11-08T02:22:48,116 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/ea344d37d002489c829b0539dc0457c4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ea344d37d002489c829b0539dc0457c4 2024-11-08T02:22:48,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741922_1098 (size=13255) 2024-11-08T02:22:48,126 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into ea344d37d002489c829b0539dc0457c4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:48,126 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:48,127 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=13, startTime=1731032568057; duration=0sec 2024-11-08T02:22:48,127 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/d20ec8239689472d90191802e9be2a42 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d20ec8239689472d90191802e9be2a42 2024-11-08T02:22:48,128 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:48,128 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:48,128 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:48,129 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:48,130 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:48,130 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:48,130 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/0dcaddcf39c04ec89572145a47f7e555, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/d13fb045ad3142e993f3b7b3eb344752, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/de6694058e2b4026a544752361437593] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=36.9 K 2024-11-08T02:22:48,130 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dcaddcf39c04ec89572145a47f7e555, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731032566612 2024-11-08T02:22:48,131 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting d13fb045ad3142e993f3b7b3eb344752, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731032566738 2024-11-08T02:22:48,151 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting de6694058e2b4026a544752361437593, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731032566883 2024-11-08T02:22:48,155 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into d20ec8239689472d90191802e9be2a42(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:48,155 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:48,155 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=13, startTime=1731032568056; duration=0sec 2024-11-08T02:22:48,155 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:48,155 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:48,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T02:22:48,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:48,161 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:22:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T02:22:48,180 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:48,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/c18d6771f6d346ab9eae05977b98be18 is 50, key is test_row_0/A:col10/1731032567595/Put/seqid=0 2024-11-08T02:22:48,181 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/54c0d16c9acd43829151b59e275aec80 is 50, key is test_row_0/C:col10/1731032567520/Put/seqid=0 2024-11-08T02:22:48,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741923_1099 (size=12301) 2024-11-08T02:22:48,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741924_1100 (size=13255) 2024-11-08T02:22:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:48,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:48,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032628241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032628246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45656 deadline: 1731032628298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:48,302 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 8469534ae513a00a18db97d665b015cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:22:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45594 deadline: 1731032628338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:48,340 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 8469534ae513a00a18db97d665b015cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:22:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032628348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032628358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032628558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032628570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,595 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/c18d6771f6d346ab9eae05977b98be18 2024-11-08T02:22:48,604 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/54c0d16c9acd43829151b59e275aec80 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/54c0d16c9acd43829151b59e275aec80 2024-11-08T02:22:48,617 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into 54c0d16c9acd43829151b59e275aec80(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:48,617 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:48,617 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=13, startTime=1731032568057; duration=0sec 2024-11-08T02:22:48,617 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:48,617 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:48,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/215accbf467b435598c44da7d3225b21 is 50, key is test_row_0/B:col10/1731032567595/Put/seqid=0 2024-11-08T02:22:48,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741925_1101 (size=12301) 2024-11-08T02:22:48,666 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/215accbf467b435598c44da7d3225b21 2024-11-08T02:22:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T02:22:48,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/e173a800cb014ac1a47db1f490c5db4f is 50, key is test_row_0/C:col10/1731032567595/Put/seqid=0 2024-11-08T02:22:48,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741926_1102 (size=12301) 2024-11-08T02:22:48,701 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/e173a800cb014ac1a47db1f490c5db4f 2024-11-08T02:22:48,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/c18d6771f6d346ab9eae05977b98be18 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/c18d6771f6d346ab9eae05977b98be18 2024-11-08T02:22:48,734 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/c18d6771f6d346ab9eae05977b98be18, entries=150, sequenceid=419, filesize=12.0 K 2024-11-08T02:22:48,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/215accbf467b435598c44da7d3225b21 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/215accbf467b435598c44da7d3225b21 2024-11-08T02:22:48,745 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/215accbf467b435598c44da7d3225b21, entries=150, sequenceid=419, filesize=12.0 K 2024-11-08T02:22:48,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/e173a800cb014ac1a47db1f490c5db4f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/e173a800cb014ac1a47db1f490c5db4f 2024-11-08T02:22:48,755 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/e173a800cb014ac1a47db1f490c5db4f, entries=150, sequenceid=419, filesize=12.0 K 2024-11-08T02:22:48,758 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8469534ae513a00a18db97d665b015cd in 597ms, sequenceid=419, compaction requested=false 2024-11-08T02:22:48,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:48,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:48,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-08T02:22:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-08T02:22:48,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-08T02:22:48,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2210 sec 2024-11-08T02:22:48,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.2400 sec 2024-11-08T02:22:48,775 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:48,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:22:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:48,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/e7f9f5bfeeff435c9c6547c88ab77b25 is 50, key is test_row_0/A:col10/1731032568772/Put/seqid=0 2024-11-08T02:22:48,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741927_1103 (size=14741) 2024-11-08T02:22:48,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/e7f9f5bfeeff435c9c6547c88ab77b25 2024-11-08T02:22:48,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/0a36b5c25c044f6185207f460b8833ed is 50, key is test_row_0/B:col10/1731032568772/Put/seqid=0 2024-11-08T02:22:48,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032628888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032628892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:48,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032628896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:48,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:48,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741928_1104 (size=12301) 2024-11-08T02:22:48,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/0a36b5c25c044f6185207f460b8833ed 2024-11-08T02:22:48,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/a8c922434d8346ddb04bb6f1797ab3e9 is 50, key is test_row_0/C:col10/1731032568772/Put/seqid=0 2024-11-08T02:22:48,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741929_1105 (size=12301) 2024-11-08T02:22:48,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/a8c922434d8346ddb04bb6f1797ab3e9 2024-11-08T02:22:49,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/e7f9f5bfeeff435c9c6547c88ab77b25 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e7f9f5bfeeff435c9c6547c88ab77b25 2024-11-08T02:22:49,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:49,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032629008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:49,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032629008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:49,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:49,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e7f9f5bfeeff435c9c6547c88ab77b25, entries=200, sequenceid=432, filesize=14.4 K 2024-11-08T02:22:49,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/0a36b5c25c044f6185207f460b8833ed as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/0a36b5c25c044f6185207f460b8833ed 2024-11-08T02:22:49,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/0a36b5c25c044f6185207f460b8833ed, entries=150, sequenceid=432, filesize=12.0 K 2024-11-08T02:22:49,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/a8c922434d8346ddb04bb6f1797ab3e9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a8c922434d8346ddb04bb6f1797ab3e9 2024-11-08T02:22:49,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a8c922434d8346ddb04bb6f1797ab3e9, entries=150, sequenceid=432, filesize=12.0 K 2024-11-08T02:22:49,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8469534ae513a00a18db97d665b015cd in 276ms, sequenceid=432, compaction requested=true 2024-11-08T02:22:49,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:49,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:49,052 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:49,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:49,052 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:49,057 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:49,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:49,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:49,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:49,058 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:49,058 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:49,058 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d20ec8239689472d90191802e9be2a42, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/c18d6771f6d346ab9eae05977b98be18, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e7f9f5bfeeff435c9c6547c88ab77b25] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=39.4 K 2024-11-08T02:22:49,059 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:49,059 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:49,059 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:49,059 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ea344d37d002489c829b0539dc0457c4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/215accbf467b435598c44da7d3225b21, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/0a36b5c25c044f6185207f460b8833ed] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=37.0 K 2024-11-08T02:22:49,060 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting d20ec8239689472d90191802e9be2a42, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731032566883 2024-11-08T02:22:49,060 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting ea344d37d002489c829b0539dc0457c4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731032566883 2024-11-08T02:22:49,061 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 215accbf467b435598c44da7d3225b21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731032567581 2024-11-08T02:22:49,062 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c18d6771f6d346ab9eae05977b98be18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731032567581 2024-11-08T02:22:49,063 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7f9f5bfeeff435c9c6547c88ab77b25, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731032568238 2024-11-08T02:22:49,063 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a36b5c25c044f6185207f460b8833ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731032568238 2024-11-08T02:22:49,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:49,091 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:49,091 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/e3780d1ed70540a9baec08063ff41658 is 50, key is test_row_0/B:col10/1731032568772/Put/seqid=0 2024-11-08T02:22:49,102 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:49,103 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/b68a0c4e8c8848958a78ba40253dcab6 is 50, key is test_row_0/A:col10/1731032568772/Put/seqid=0 2024-11-08T02:22:49,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741930_1106 (size=13357) 2024-11-08T02:22:49,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741931_1107 (size=13357) 2024-11-08T02:22:49,189 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/e3780d1ed70540a9baec08063ff41658 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e3780d1ed70540a9baec08063ff41658 2024-11-08T02:22:49,202 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into e3780d1ed70540a9baec08063ff41658(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:49,202 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:49,203 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=13, startTime=1731032569052; duration=0sec 2024-11-08T02:22:49,203 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:49,203 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:49,203 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:49,206 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:49,206 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:49,206 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:49,209 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/54c0d16c9acd43829151b59e275aec80, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/e173a800cb014ac1a47db1f490c5db4f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a8c922434d8346ddb04bb6f1797ab3e9] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=37.0 K 2024-11-08T02:22:49,213 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 54c0d16c9acd43829151b59e275aec80, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731032566883 2024-11-08T02:22:49,213 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting e173a800cb014ac1a47db1f490c5db4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731032567581 2024-11-08T02:22:49,214 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a8c922434d8346ddb04bb6f1797ab3e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731032568238 2024-11-08T02:22:49,230 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:49,231 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:49,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:22:49,232 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/044f3ba17a194a56a25ee4826cb69953 is 50, key is test_row_0/C:col10/1731032568772/Put/seqid=0 2024-11-08T02:22:49,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/9d5d60c5dabe4f7eafd20209c9aece61 is 50, key is test_row_0/A:col10/1731032569223/Put/seqid=0 2024-11-08T02:22:49,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:49,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741932_1108 (size=17181) 2024-11-08T02:22:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032629273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032629273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:49,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741933_1109 (size=13357) 2024-11-08T02:22:49,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032629388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:49,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032629393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:49,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032629397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:49,554 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/b68a0c4e8c8848958a78ba40253dcab6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/b68a0c4e8c8848958a78ba40253dcab6 2024-11-08T02:22:49,562 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into b68a0c4e8c8848958a78ba40253dcab6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:49,563 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:49,563 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=13, startTime=1731032569051; duration=0sec 2024-11-08T02:22:49,565 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:49,565 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032629598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032629598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:49,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/9d5d60c5dabe4f7eafd20209c9aece61 2024-11-08T02:22:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T02:22:49,689 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:22:49,689 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:22:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-08T02:22:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T02:22:49,693 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:22:49,694 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:22:49,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:22:49,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/5b55ac1ae38d4ed1bcb01c70b88b6207 is 50, key is test_row_0/B:col10/1731032569223/Put/seqid=0 2024-11-08T02:22:49,716 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/044f3ba17a194a56a25ee4826cb69953 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/044f3ba17a194a56a25ee4826cb69953 2024-11-08T02:22:49,724 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into 044f3ba17a194a56a25ee4826cb69953(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:49,724 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:49,725 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=13, startTime=1731032569057; duration=0sec 2024-11-08T02:22:49,725 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:49,725 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:49,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741934_1110 (size=12301) 2024-11-08T02:22:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T02:22:49,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T02:22:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:49,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:49,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032629908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032629908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:49,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:50,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T02:22:50,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:50,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T02:22:50,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/5b55ac1ae38d4ed1bcb01c70b88b6207 2024-11-08T02:22:50,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T02:22:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/9f08d6c003194636bbf8c60846a7595a is 50, key is test_row_0/C:col10/1731032569223/Put/seqid=0 2024-11-08T02:22:50,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741935_1111 (size=12301) 2024-11-08T02:22:50,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T02:22:50,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:50,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T02:22:50,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:50,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032630408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:50,417 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 8469534ae513a00a18db97d665b015cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:22:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:50,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:50,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032630418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:50,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:50,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032630419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:50,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T02:22:50,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/9f08d6c003194636bbf8c60846a7595a 2024-11-08T02:22:50,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/9d5d60c5dabe4f7eafd20209c9aece61 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9d5d60c5dabe4f7eafd20209c9aece61 2024-11-08T02:22:50,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T02:22:50,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:50,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:50,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9d5d60c5dabe4f7eafd20209c9aece61, entries=250, sequenceid=459, filesize=16.8 K 2024-11-08T02:22:50,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/5b55ac1ae38d4ed1bcb01c70b88b6207 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/5b55ac1ae38d4ed1bcb01c70b88b6207 2024-11-08T02:22:50,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/5b55ac1ae38d4ed1bcb01c70b88b6207, entries=150, sequenceid=459, filesize=12.0 K 2024-11-08T02:22:50,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/9f08d6c003194636bbf8c60846a7595a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/9f08d6c003194636bbf8c60846a7595a 2024-11-08T02:22:50,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/9f08d6c003194636bbf8c60846a7595a, entries=150, sequenceid=459, filesize=12.0 K 2024-11-08T02:22:50,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8469534ae513a00a18db97d665b015cd in 1416ms, sequenceid=459, compaction requested=false 2024-11-08T02:22:50,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:50,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T02:22:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,778 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T02:22:50,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/693403b20e9f4dfdb01e19b4b325c72b is 50, key is test_row_0/A:col10/1731032569269/Put/seqid=0 2024-11-08T02:22:50,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741936_1112 (size=12301) 2024-11-08T02:22:50,810 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/693403b20e9f4dfdb01e19b4b325c72b 2024-11-08T02:22:50,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/ce3cbc6bb9f747b48075ba7db4b86f04 is 50, key is test_row_0/B:col10/1731032569269/Put/seqid=0 2024-11-08T02:22:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T02:22:50,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741937_1113 (size=12301) 2024-11-08T02:22:50,848 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/ce3cbc6bb9f747b48075ba7db4b86f04 2024-11-08T02:22:50,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/fda6eb80b08b431bbe4cfb662943aa86 is 50, key is test_row_0/C:col10/1731032569269/Put/seqid=0 2024-11-08T02:22:50,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741938_1114 (size=12301) 2024-11-08T02:22:50,895 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/fda6eb80b08b431bbe4cfb662943aa86 2024-11-08T02:22:50,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/693403b20e9f4dfdb01e19b4b325c72b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/693403b20e9f4dfdb01e19b4b325c72b 2024-11-08T02:22:50,912 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/693403b20e9f4dfdb01e19b4b325c72b, entries=150, sequenceid=471, filesize=12.0 K 2024-11-08T02:22:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/ce3cbc6bb9f747b48075ba7db4b86f04 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ce3cbc6bb9f747b48075ba7db4b86f04 2024-11-08T02:22:50,925 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ce3cbc6bb9f747b48075ba7db4b86f04, entries=150, sequenceid=471, filesize=12.0 K 2024-11-08T02:22:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/fda6eb80b08b431bbe4cfb662943aa86 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/fda6eb80b08b431bbe4cfb662943aa86 2024-11-08T02:22:50,942 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/fda6eb80b08b431bbe4cfb662943aa86, entries=150, sequenceid=471, filesize=12.0 K 2024-11-08T02:22:50,946 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 8469534ae513a00a18db97d665b015cd in 168ms, sequenceid=471, compaction requested=true 2024-11-08T02:22:50,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:50,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:50,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-08T02:22:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-08T02:22:50,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-08T02:22:50,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2540 sec 2024-11-08T02:22:50,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2630 sec 2024-11-08T02:22:51,465 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:51,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:22:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:51,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/e509d4d02fd641d59dab07e87a2c57a5 is 50, key is test_row_0/A:col10/1731032571459/Put/seqid=0 2024-11-08T02:22:51,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741939_1115 (size=17181) 2024-11-08T02:22:51,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/e509d4d02fd641d59dab07e87a2c57a5 2024-11-08T02:22:51,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/58e482fb2c8140be8a9b72110b45bf99 is 50, key is test_row_0/B:col10/1731032571459/Put/seqid=0 2024-11-08T02:22:51,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032631617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:51,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032631617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:51,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741940_1116 (size=12301) 2024-11-08T02:22:51,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032631738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:51,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032631751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T02:22:51,848 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:22:51,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:22:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-08T02:22:51,852 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:22:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T02:22:51,854 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:22:51,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:22:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T02:22:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032631958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032631958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:52,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T02:22:52,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:52,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:52,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:52,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:52,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:52,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/58e482fb2c8140be8a9b72110b45bf99 2024-11-08T02:22:52,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/077d8ffe2b4241958db48df40f544f00 is 50, key is test_row_0/C:col10/1731032571459/Put/seqid=0 2024-11-08T02:22:52,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741941_1117 (size=12301) 2024-11-08T02:22:52,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/077d8ffe2b4241958db48df40f544f00 2024-11-08T02:22:52,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/e509d4d02fd641d59dab07e87a2c57a5 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e509d4d02fd641d59dab07e87a2c57a5 2024-11-08T02:22:52,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e509d4d02fd641d59dab07e87a2c57a5, entries=250, sequenceid=483, filesize=16.8 K 2024-11-08T02:22:52,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/58e482fb2c8140be8a9b72110b45bf99 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/58e482fb2c8140be8a9b72110b45bf99 2024-11-08T02:22:52,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/58e482fb2c8140be8a9b72110b45bf99, entries=150, sequenceid=483, filesize=12.0 K 2024-11-08T02:22:52,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/077d8ffe2b4241958db48df40f544f00 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/077d8ffe2b4241958db48df40f544f00 2024-11-08T02:22:52,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/077d8ffe2b4241958db48df40f544f00, entries=150, sequenceid=483, filesize=12.0 K 2024-11-08T02:22:52,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 8469534ae513a00a18db97d665b015cd in 663ms, sequenceid=483, compaction requested=true 2024-11-08T02:22:52,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:52,129 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:22:52,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:52,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:52,131 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:22:52,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:52,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:52,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:52,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:52,133 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 60020 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:22:52,133 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:52,133 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:52,133 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/b68a0c4e8c8848958a78ba40253dcab6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9d5d60c5dabe4f7eafd20209c9aece61, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/693403b20e9f4dfdb01e19b4b325c72b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e509d4d02fd641d59dab07e87a2c57a5] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=58.6 K 2024-11-08T02:22:52,134 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b68a0c4e8c8848958a78ba40253dcab6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731032568238 2024-11-08T02:22:52,134 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d5d60c5dabe4f7eafd20209c9aece61, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731032568874 2024-11-08T02:22:52,134 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:22:52,134 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:52,135 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:52,135 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e3780d1ed70540a9baec08063ff41658, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/5b55ac1ae38d4ed1bcb01c70b88b6207, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ce3cbc6bb9f747b48075ba7db4b86f04, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/58e482fb2c8140be8a9b72110b45bf99] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=49.1 K 2024-11-08T02:22:52,135 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting e3780d1ed70540a9baec08063ff41658, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731032568238 2024-11-08T02:22:52,136 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 693403b20e9f4dfdb01e19b4b325c72b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731032569246 2024-11-08T02:22:52,136 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b55ac1ae38d4ed1bcb01c70b88b6207, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731032568891 2024-11-08T02:22:52,136 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting e509d4d02fd641d59dab07e87a2c57a5, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731032571446 2024-11-08T02:22:52,137 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting ce3cbc6bb9f747b48075ba7db4b86f04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731032569246 2024-11-08T02:22:52,137 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e482fb2c8140be8a9b72110b45bf99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731032571446 2024-11-08T02:22:52,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T02:22:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:52,166 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:22:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T02:22:52,176 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#105 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:52,176 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/23dabb1a215b4009b4c0a2f8f18e4283 is 50, key is test_row_0/B:col10/1731032571459/Put/seqid=0 2024-11-08T02:22:52,178 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:52,179 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/d77a6eb4aa7d40fd88aa185fa27a5fc6 is 50, key is test_row_0/A:col10/1731032571459/Put/seqid=0 2024-11-08T02:22:52,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/0655523e4be747ddbba60caaa903c634 is 50, key is test_row_0/A:col10/1731032571592/Put/seqid=0 2024-11-08T02:22:52,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741942_1118 (size=13493) 2024-11-08T02:22:52,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741943_1119 (size=13493) 2024-11-08T02:22:52,234 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/d77a6eb4aa7d40fd88aa185fa27a5fc6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d77a6eb4aa7d40fd88aa185fa27a5fc6 2024-11-08T02:22:52,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741944_1120 (size=12301) 2024-11-08T02:22:52,236 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/0655523e4be747ddbba60caaa903c634 2024-11-08T02:22:52,248 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into d77a6eb4aa7d40fd88aa185fa27a5fc6(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:52,248 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:52,248 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=12, startTime=1731032572129; duration=0sec 2024-11-08T02:22:52,248 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:52,248 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:52,249 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:22:52,251 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:22:52,252 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:52,252 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:52,252 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/044f3ba17a194a56a25ee4826cb69953, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/9f08d6c003194636bbf8c60846a7595a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/fda6eb80b08b431bbe4cfb662943aa86, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/077d8ffe2b4241958db48df40f544f00] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=49.1 K 2024-11-08T02:22:52,252 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 044f3ba17a194a56a25ee4826cb69953, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731032568238 2024-11-08T02:22:52,253 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f08d6c003194636bbf8c60846a7595a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731032568891 2024-11-08T02:22:52,253 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting fda6eb80b08b431bbe4cfb662943aa86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731032569246 2024-11-08T02:22:52,254 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 077d8ffe2b4241958db48df40f544f00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731032571446 2024-11-08T02:22:52,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/845e04505a554b3a859b73380f52d5ed is 50, key is test_row_0/B:col10/1731032571592/Put/seqid=0 2024-11-08T02:22:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:52,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:52,299 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:52,300 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/8cbdab21d4534b82bb7141025fee63bb is 50, key is test_row_0/C:col10/1731032571459/Put/seqid=0 2024-11-08T02:22:52,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741945_1121 (size=12301) 2024-11-08T02:22:52,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741946_1122 (size=13493) 2024-11-08T02:22:52,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032632335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:52,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032632343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:52,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45612 deadline: 1731032632428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:52,431 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 8469534ae513a00a18db97d665b015cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:22:52,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:52,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032632449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:52,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032632449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T02:22:52,629 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/23dabb1a215b4009b4c0a2f8f18e4283 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/23dabb1a215b4009b4c0a2f8f18e4283 2024-11-08T02:22:52,638 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into 23dabb1a215b4009b4c0a2f8f18e4283(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:52,638 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:52,638 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=12, startTime=1731032572131; duration=0sec 2024-11-08T02:22:52,639 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:52,639 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:52,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032632658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032632659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:52,705 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/845e04505a554b3a859b73380f52d5ed 2024-11-08T02:22:52,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/928918b4ed134b85a80dc602e82838c4 is 50, key is test_row_0/C:col10/1731032571592/Put/seqid=0 2024-11-08T02:22:52,737 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/8cbdab21d4534b82bb7141025fee63bb as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/8cbdab21d4534b82bb7141025fee63bb 2024-11-08T02:22:52,749 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into 8cbdab21d4534b82bb7141025fee63bb(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:52,749 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:52,749 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=12, startTime=1731032572131; duration=0sec 2024-11-08T02:22:52,749 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:52,749 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:52,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741947_1123 (size=12301) 2024-11-08T02:22:52,785 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/928918b4ed134b85a80dc602e82838c4 2024-11-08T02:22:52,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/0655523e4be747ddbba60caaa903c634 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/0655523e4be747ddbba60caaa903c634 2024-11-08T02:22:52,804 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/0655523e4be747ddbba60caaa903c634, entries=150, sequenceid=508, filesize=12.0 K 2024-11-08T02:22:52,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/845e04505a554b3a859b73380f52d5ed as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/845e04505a554b3a859b73380f52d5ed 2024-11-08T02:22:52,814 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/845e04505a554b3a859b73380f52d5ed, entries=150, sequenceid=508, filesize=12.0 K 2024-11-08T02:22:52,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/928918b4ed134b85a80dc602e82838c4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/928918b4ed134b85a80dc602e82838c4 2024-11-08T02:22:52,825 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/928918b4ed134b85a80dc602e82838c4, entries=150, sequenceid=508, filesize=12.0 K 2024-11-08T02:22:52,826 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8469534ae513a00a18db97d665b015cd in 661ms, sequenceid=508, compaction requested=false 2024-11-08T02:22:52,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:52,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:52,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-08T02:22:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-08T02:22:52,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-08T02:22:52,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 974 msec 2024-11-08T02:22:52,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 982 msec 2024-11-08T02:22:52,970 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:52,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:22:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:52,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/b813115baa9e4451b55cde7f2fb1a115 is 50, key is test_row_0/A:col10/1731032572968/Put/seqid=0 2024-11-08T02:22:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T02:22:52,989 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:22:52,989 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:22:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-08T02:22:52,991 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:22:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T02:22:52,992 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:22:52,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:22:53,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741948_1124 (size=12301) 2024-11-08T02:22:53,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/b813115baa9e4451b55cde7f2fb1a115 2024-11-08T02:22:53,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/e8d0922176a9429ca43c2d4949fa13ae is 50, key is test_row_0/B:col10/1731032572968/Put/seqid=0 2024-11-08T02:22:53,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032633060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:53,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:53,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032633062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T02:22:53,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741949_1125 (size=12301) 2024-11-08T02:22:53,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:53,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:53,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032633178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:53,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032633178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:53,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T02:22:53,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032633388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032633388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:53,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:53,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:53,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:53,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/e8d0922176a9429ca43c2d4949fa13ae 2024-11-08T02:22:53,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/b1ac23c5878f465c920b8e637152f42d is 50, key is test_row_0/C:col10/1731032572968/Put/seqid=0 2024-11-08T02:22:53,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741950_1126 (size=12301) 2024-11-08T02:22:53,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/b1ac23c5878f465c920b8e637152f42d 2024-11-08T02:22:53,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/b813115baa9e4451b55cde7f2fb1a115 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/b813115baa9e4451b55cde7f2fb1a115 2024-11-08T02:22:53,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/b813115baa9e4451b55cde7f2fb1a115, entries=150, sequenceid=523, filesize=12.0 K 2024-11-08T02:22:53,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/e8d0922176a9429ca43c2d4949fa13ae as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e8d0922176a9429ca43c2d4949fa13ae 2024-11-08T02:22:53,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e8d0922176a9429ca43c2d4949fa13ae, entries=150, sequenceid=523, filesize=12.0 K 2024-11-08T02:22:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T02:22:53,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/b1ac23c5878f465c920b8e637152f42d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b1ac23c5878f465c920b8e637152f42d 2024-11-08T02:22:53,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:53,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b1ac23c5878f465c920b8e637152f42d, entries=150, sequenceid=523, filesize=12.0 K 2024-11-08T02:22:53,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8469534ae513a00a18db97d665b015cd in 661ms, sequenceid=523, compaction requested=true 2024-11-08T02:22:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:53,632 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:53,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:22:53,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:53,632 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:53,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:22:53,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:53,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8469534ae513a00a18db97d665b015cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:22:53,633 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:53,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:53,634 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/A is initiating minor compaction (all files) 2024-11-08T02:22:53,634 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/A in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,634 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d77a6eb4aa7d40fd88aa185fa27a5fc6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/0655523e4be747ddbba60caaa903c634, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/b813115baa9e4451b55cde7f2fb1a115] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=37.2 K 2024-11-08T02:22:53,634 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:53,634 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/B is initiating minor compaction (all files) 2024-11-08T02:22:53,634 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/B in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,635 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/23dabb1a215b4009b4c0a2f8f18e4283, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/845e04505a554b3a859b73380f52d5ed, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e8d0922176a9429ca43c2d4949fa13ae] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=37.2 K 2024-11-08T02:22:53,635 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting d77a6eb4aa7d40fd88aa185fa27a5fc6, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731032571446 2024-11-08T02:22:53,635 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 23dabb1a215b4009b4c0a2f8f18e4283, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731032571446 2024-11-08T02:22:53,635 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0655523e4be747ddbba60caaa903c634, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731032571582 2024-11-08T02:22:53,636 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 845e04505a554b3a859b73380f52d5ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731032571582 2024-11-08T02:22:53,636 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b813115baa9e4451b55cde7f2fb1a115, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731032572274 2024-11-08T02:22:53,636 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting e8d0922176a9429ca43c2d4949fa13ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731032572274 2024-11-08T02:22:53,647 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#B#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:53,648 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/b27c1956066c4f5fa5e29a4c23202549 is 50, key is test_row_0/B:col10/1731032572968/Put/seqid=0 2024-11-08T02:22:53,650 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#A#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:53,650 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/65210d00e424465f9a0284507ca7d837 is 50, key is test_row_0/A:col10/1731032572968/Put/seqid=0 2024-11-08T02:22:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:53,705 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:22:53,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:22:53,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741951_1127 (size=13595) 2024-11-08T02:22:53,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032633741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:53,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:53,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741952_1128 (size=13595) 2024-11-08T02:22:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032633745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:53,759 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/65210d00e424465f9a0284507ca7d837 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/65210d00e424465f9a0284507ca7d837 2024-11-08T02:22:53,766 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/A of 8469534ae513a00a18db97d665b015cd into 65210d00e424465f9a0284507ca7d837(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:53,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/ff7135ba2b1349869e1f57d0c1049625 is 50, key is test_row_0/A:col10/1731032573060/Put/seqid=0 2024-11-08T02:22:53,766 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:53,766 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/A, priority=13, startTime=1731032573631; duration=0sec 2024-11-08T02:22:53,767 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:22:53,767 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:A 2024-11-08T02:22:53,767 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:22:53,769 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:22:53,769 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 8469534ae513a00a18db97d665b015cd/C is initiating minor compaction (all files) 2024-11-08T02:22:53,769 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8469534ae513a00a18db97d665b015cd/C in TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,769 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/8cbdab21d4534b82bb7141025fee63bb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/928918b4ed134b85a80dc602e82838c4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b1ac23c5878f465c920b8e637152f42d] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp, totalSize=37.2 K 2024-11-08T02:22:53,770 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cbdab21d4534b82bb7141025fee63bb, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731032571446 2024-11-08T02:22:53,771 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 928918b4ed134b85a80dc602e82838c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731032571582 2024-11-08T02:22:53,771 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1ac23c5878f465c920b8e637152f42d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731032572274 2024-11-08T02:22:53,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741953_1129 (size=17181) 2024-11-08T02:22:53,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,789 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8469534ae513a00a18db97d665b015cd#C#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:22:53,790 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/5a337572870a47a393f8ca32144c364c is 50, key is test_row_0/C:col10/1731032572968/Put/seqid=0 2024-11-08T02:22:53,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741954_1130 (size=13595) 2024-11-08T02:22:53,829 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/5a337572870a47a393f8ca32144c364c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/5a337572870a47a393f8ca32144c364c 2024-11-08T02:22:53,844 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/C of 8469534ae513a00a18db97d665b015cd into 5a337572870a47a393f8ca32144c364c(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:53,844 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:53,844 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/C, priority=13, startTime=1731032573633; duration=0sec 2024-11-08T02:22:53,844 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:53,845 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:C 2024-11-08T02:22:53,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032633848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:53,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032633859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:53,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:53,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:53,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:53,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032634058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:54,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:54,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032634068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:54,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:54,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:54,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,124 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/b27c1956066c4f5fa5e29a4c23202549 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/b27c1956066c4f5fa5e29a4c23202549 2024-11-08T02:22:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T02:22:54,131 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8469534ae513a00a18db97d665b015cd/B of 8469534ae513a00a18db97d665b015cd into b27c1956066c4f5fa5e29a4c23202549(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:22:54,131 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:54,131 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., storeName=8469534ae513a00a18db97d665b015cd/B, priority=13, startTime=1731032573632; duration=0sec 2024-11-08T02:22:54,133 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:22:54,133 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8469534ae513a00a18db97d665b015cd:B 2024-11-08T02:22:54,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/ff7135ba2b1349869e1f57d0c1049625 2024-11-08T02:22:54,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/4ac224888450478d9902df6fad4fb142 is 50, key is test_row_0/B:col10/1731032573060/Put/seqid=0 2024-11-08T02:22:54,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:54,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:54,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741955_1131 (size=12301) 2024-11-08T02:22:54,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032634368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:54,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032634378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:54,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:54,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:54,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:54,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:54,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/4ac224888450478d9902df6fad4fb142 2024-11-08T02:22:54,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/dff8ab9f4b3945e88e4de40ddf5ac27a is 50, key is test_row_0/C:col10/1731032573060/Put/seqid=0 2024-11-08T02:22:54,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741956_1132 (size=12301) 2024-11-08T02:22:54,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:54,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:54,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:54,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:54,870 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-08T02:22:54,870 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:22:54,871 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-08T02:22:54,871 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:22:54,876 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-08T02:22:54,876 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:54,876 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:22:54,876 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:54,876 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:54,877 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:54,877 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:54,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:22:54,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:22:54,877 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:54,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:22:54,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:54,879 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-08T02:22:54,879 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:22:54,879 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45630 deadline: 1731032634878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:54,879 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:54,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:22:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:54,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:22:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45592 deadline: 1731032634898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 2024-11-08T02:22:54,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:22:54,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:22:55,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:55,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:22:55,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/dff8ab9f4b3945e88e4de40ddf5ac27a 2024-11-08T02:22:55,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/ff7135ba2b1349869e1f57d0c1049625 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/ff7135ba2b1349869e1f57d0c1049625 2024-11-08T02:22:55,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/ff7135ba2b1349869e1f57d0c1049625, entries=250, sequenceid=549, filesize=16.8 K 2024-11-08T02:22:55,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/4ac224888450478d9902df6fad4fb142 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4ac224888450478d9902df6fad4fb142 2024-11-08T02:22:55,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4ac224888450478d9902df6fad4fb142, entries=150, sequenceid=549, filesize=12.0 K 2024-11-08T02:22:55,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/dff8ab9f4b3945e88e4de40ddf5ac27a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/dff8ab9f4b3945e88e4de40ddf5ac27a 2024-11-08T02:22:55,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/dff8ab9f4b3945e88e4de40ddf5ac27a, entries=150, sequenceid=549, filesize=12.0 K 2024-11-08T02:22:55,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8469534ae513a00a18db97d665b015cd in 1430ms, sequenceid=549, compaction requested=false 2024-11-08T02:22:55,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T02:22:55,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T02:22:55,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:55,168 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T02:22:55,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/7144498580b54918bdf3373d9b90c275 is 50, key is test_row_0/A:col10/1731032573723/Put/seqid=0 2024-11-08T02:22:55,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741957_1133 (size=12301) 2024-11-08T02:22:55,578 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/7144498580b54918bdf3373d9b90c275 2024-11-08T02:22:55,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/3f0bf9f5ab8246b8a314159ad5cc5bcc is 50, key is test_row_0/B:col10/1731032573723/Put/seqid=0 2024-11-08T02:22:55,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741958_1134 (size=12301) 2024-11-08T02:22:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:55,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. as already flushing 2024-11-08T02:22:55,900 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-08T02:22:55,900 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:22:55,900 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:55,900 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:55,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:22:55,909 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-08T02:22:55,909 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:22:55,909 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:55,909 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:55,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:22:55,992 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/3f0bf9f5ab8246b8a314159ad5cc5bcc 2024-11-08T02:22:56,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/a2e934eb16ff467ea2d856df556d51f6 is 50, key is test_row_0/C:col10/1731032573723/Put/seqid=0 2024-11-08T02:22:56,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741959_1135 (size=12301) 2024-11-08T02:22:56,405 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/a2e934eb16ff467ea2d856df556d51f6 2024-11-08T02:22:56,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/7144498580b54918bdf3373d9b90c275 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7144498580b54918bdf3373d9b90c275 2024-11-08T02:22:56,415 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7144498580b54918bdf3373d9b90c275, entries=150, sequenceid=562, filesize=12.0 K 2024-11-08T02:22:56,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/3f0bf9f5ab8246b8a314159ad5cc5bcc as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/3f0bf9f5ab8246b8a314159ad5cc5bcc 2024-11-08T02:22:56,421 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/3f0bf9f5ab8246b8a314159ad5cc5bcc, entries=150, sequenceid=562, filesize=12.0 K 2024-11-08T02:22:56,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/a2e934eb16ff467ea2d856df556d51f6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a2e934eb16ff467ea2d856df556d51f6 2024-11-08T02:22:56,426 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a2e934eb16ff467ea2d856df556d51f6, entries=150, sequenceid=562, filesize=12.0 K 2024-11-08T02:22:56,427 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=13.42 KB/13740 for 8469534ae513a00a18db97d665b015cd in 1260ms, sequenceid=562, compaction requested=true 2024-11-08T02:22:56,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 8469534ae513a00a18db97d665b015cd: 2024-11-08T02:22:56,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:56,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-08T02:22:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-08T02:22:56,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-08T02:22:56,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4360 sec 2024-11-08T02:22:56,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 3.4420 sec 2024-11-08T02:22:56,449 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-08T02:22:56,449 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:22:56,450 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:56,450 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:56,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:22:56,569 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:22:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T02:22:57,148 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:22:58,399 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-08T02:22:58,399 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:22:58,400 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:58,400 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:58,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:22:58,463 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-08T02:22:58,463 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:22:58,463 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:58,463 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:58,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 165 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3184 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3061 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1444 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4330 rows 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1449 2024-11-08T02:22:58,464 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4339 rows 2024-11-08T02:22:58,464 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T02:22:58,464 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:22:58,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:58,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:22:58,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:22:58,469 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:22:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:22:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T02:22:58,481 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032578480"}]},"ts":"1731032578480"} 2024-11-08T02:22:58,483 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:22:58,483 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:22:58,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:22:58,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8469534ae513a00a18db97d665b015cd, UNASSIGN}] 2024-11-08T02:22:58,492 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8469534ae513a00a18db97d665b015cd, UNASSIGN 2024-11-08T02:22:58,496 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=8469534ae513a00a18db97d665b015cd, regionState=CLOSING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:22:58,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8469534ae513a00a18db97d665b015cd, UNASSIGN because future has completed 2024-11-08T02:22:58,500 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:22:58,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:22:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T02:22:58,659 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 8469534ae513a00a18db97d665b015cd 2024-11-08T02:22:58,659 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T02:22:58,660 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 8469534ae513a00a18db97d665b015cd, disabling compactions & flushes 2024-11-08T02:22:58,660 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:58,660 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:58,660 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. after waiting 0 ms 2024-11-08T02:22:58,660 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:22:58,660 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 8469534ae513a00a18db97d665b015cd 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T02:22:58,666 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/1ab1e3647aa041d8a367b3e928ff9f3b is 50, key is test_row_0/A:col10/1731032576448/Put/seqid=0 2024-11-08T02:22:58,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741960_1136 (size=12301) 2024-11-08T02:22:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T02:22:59,073 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/1ab1e3647aa041d8a367b3e928ff9f3b 2024-11-08T02:22:59,090 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/9bd7b9efb6d9410bb3f1e23c797c21ed is 50, key is test_row_0/B:col10/1731032576448/Put/seqid=0 2024-11-08T02:22:59,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741961_1137 (size=12301) 2024-11-08T02:22:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T02:22:59,500 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/9bd7b9efb6d9410bb3f1e23c797c21ed 2024-11-08T02:22:59,508 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/75b2c370c6904b4c9a7b3d575551d72e is 50, key is test_row_0/C:col10/1731032576448/Put/seqid=0 2024-11-08T02:22:59,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741962_1138 (size=12301) 2024-11-08T02:22:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T02:22:59,912 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/75b2c370c6904b4c9a7b3d575551d72e 2024-11-08T02:22:59,918 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/A/1ab1e3647aa041d8a367b3e928ff9f3b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1ab1e3647aa041d8a367b3e928ff9f3b 2024-11-08T02:22:59,924 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1ab1e3647aa041d8a367b3e928ff9f3b, entries=150, sequenceid=570, filesize=12.0 K 2024-11-08T02:22:59,925 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/B/9bd7b9efb6d9410bb3f1e23c797c21ed as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/9bd7b9efb6d9410bb3f1e23c797c21ed 2024-11-08T02:22:59,933 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/9bd7b9efb6d9410bb3f1e23c797c21ed, entries=150, sequenceid=570, filesize=12.0 K 2024-11-08T02:22:59,934 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/.tmp/C/75b2c370c6904b4c9a7b3d575551d72e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/75b2c370c6904b4c9a7b3d575551d72e 2024-11-08T02:22:59,941 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/75b2c370c6904b4c9a7b3d575551d72e, entries=150, sequenceid=570, filesize=12.0 K 2024-11-08T02:22:59,942 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8469534ae513a00a18db97d665b015cd in 1282ms, sequenceid=570, compaction requested=true 2024-11-08T02:22:59,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7aac23f2d00a4d91b607113ed6657f32, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d88ef76fb40d4347b7b7842bdbd33e9d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/54867e182d3e4ffb9a688ae169aeac0d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9ca585a1529147439680c4313d55f658, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fb94f483c980415f982d000902b7f85f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7d2c545870c547da932d151c0260d391, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9f4ee173de6444279ab59235f1ee4a51, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/79129cdadc8046fea9c26c49b78a6959, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/a2c2c41dc9a444e5abb192bfff0b2cc0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e89b2708699d4f108082bd36737cb678, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/332f2b1131be4c318aa7f0bc354bd3bf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/57661a2e566d49d48ecc52e9ecb339ff, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2c99101b73bf433a990119f4de3c8d99, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1bb84ee74b95426cb451c63037d44dd0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/8b4c0d73b89c438783e9b749e3245fad, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/adea0ce45c464738ae3715ee46968d25, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/caa8d8b9ba504c3899bb9aedf890ec24, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/04adfd40ced54a6aa1649d546e0525eb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fd9a9304f3634d46afc243c74f368d18, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/633bd126d2704c8fa7630b476a6c930a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/11e3c0a23c85403b8abd03f3f993f089, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/85b043222c3f47ebb15d600f1f303673, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1999f6235f6e45f59b0f03cf792daabb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/6b8eb9202c754176b00e66a25301f0c9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/ad7e55a76da54a7b9e67e69c17b7176c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2a7343ad4a084c4f9117c37745888328, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/3b77938ab0824d9296ef77b8af8c9f14, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d20ec8239689472d90191802e9be2a42, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e071280e00cc4414ad9bee76847601f8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/c18d6771f6d346ab9eae05977b98be18, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e7f9f5bfeeff435c9c6547c88ab77b25, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/b68a0c4e8c8848958a78ba40253dcab6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9d5d60c5dabe4f7eafd20209c9aece61, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/693403b20e9f4dfdb01e19b4b325c72b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e509d4d02fd641d59dab07e87a2c57a5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d77a6eb4aa7d40fd88aa185fa27a5fc6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/0655523e4be747ddbba60caaa903c634, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/b813115baa9e4451b55cde7f2fb1a115] to archive 2024-11-08T02:22:59,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:22:59,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7aac23f2d00a4d91b607113ed6657f32 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7aac23f2d00a4d91b607113ed6657f32 2024-11-08T02:22:59,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d88ef76fb40d4347b7b7842bdbd33e9d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d88ef76fb40d4347b7b7842bdbd33e9d 2024-11-08T02:22:59,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/54867e182d3e4ffb9a688ae169aeac0d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/54867e182d3e4ffb9a688ae169aeac0d 2024-11-08T02:22:59,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9ca585a1529147439680c4313d55f658 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9ca585a1529147439680c4313d55f658 2024-11-08T02:22:59,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fb94f483c980415f982d000902b7f85f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fb94f483c980415f982d000902b7f85f 2024-11-08T02:22:59,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7d2c545870c547da932d151c0260d391 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7d2c545870c547da932d151c0260d391 2024-11-08T02:22:59,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9f4ee173de6444279ab59235f1ee4a51 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9f4ee173de6444279ab59235f1ee4a51 2024-11-08T02:22:59,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/79129cdadc8046fea9c26c49b78a6959 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/79129cdadc8046fea9c26c49b78a6959 2024-11-08T02:22:59,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/a2c2c41dc9a444e5abb192bfff0b2cc0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/a2c2c41dc9a444e5abb192bfff0b2cc0 2024-11-08T02:22:59,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e89b2708699d4f108082bd36737cb678 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e89b2708699d4f108082bd36737cb678 2024-11-08T02:22:59,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/332f2b1131be4c318aa7f0bc354bd3bf to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/332f2b1131be4c318aa7f0bc354bd3bf 2024-11-08T02:22:59,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/57661a2e566d49d48ecc52e9ecb339ff to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/57661a2e566d49d48ecc52e9ecb339ff 2024-11-08T02:22:59,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2c99101b73bf433a990119f4de3c8d99 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2c99101b73bf433a990119f4de3c8d99 2024-11-08T02:22:59,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1bb84ee74b95426cb451c63037d44dd0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1bb84ee74b95426cb451c63037d44dd0 2024-11-08T02:22:59,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/8b4c0d73b89c438783e9b749e3245fad to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/8b4c0d73b89c438783e9b749e3245fad 2024-11-08T02:22:59,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/adea0ce45c464738ae3715ee46968d25 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/adea0ce45c464738ae3715ee46968d25 2024-11-08T02:22:59,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/caa8d8b9ba504c3899bb9aedf890ec24 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/caa8d8b9ba504c3899bb9aedf890ec24 2024-11-08T02:22:59,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/04adfd40ced54a6aa1649d546e0525eb to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/04adfd40ced54a6aa1649d546e0525eb 2024-11-08T02:22:59,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fd9a9304f3634d46afc243c74f368d18 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/fd9a9304f3634d46afc243c74f368d18 2024-11-08T02:22:59,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/633bd126d2704c8fa7630b476a6c930a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/633bd126d2704c8fa7630b476a6c930a 2024-11-08T02:22:59,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/11e3c0a23c85403b8abd03f3f993f089 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/11e3c0a23c85403b8abd03f3f993f089 2024-11-08T02:22:59,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/85b043222c3f47ebb15d600f1f303673 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/85b043222c3f47ebb15d600f1f303673 2024-11-08T02:22:59,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1999f6235f6e45f59b0f03cf792daabb to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1999f6235f6e45f59b0f03cf792daabb 2024-11-08T02:22:59,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/6b8eb9202c754176b00e66a25301f0c9 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/6b8eb9202c754176b00e66a25301f0c9 2024-11-08T02:22:59,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/ad7e55a76da54a7b9e67e69c17b7176c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/ad7e55a76da54a7b9e67e69c17b7176c 2024-11-08T02:22:59,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2a7343ad4a084c4f9117c37745888328 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/2a7343ad4a084c4f9117c37745888328 2024-11-08T02:22:59,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/3b77938ab0824d9296ef77b8af8c9f14 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/3b77938ab0824d9296ef77b8af8c9f14 2024-11-08T02:22:59,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d20ec8239689472d90191802e9be2a42 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d20ec8239689472d90191802e9be2a42 2024-11-08T02:22:59,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e071280e00cc4414ad9bee76847601f8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e071280e00cc4414ad9bee76847601f8 2024-11-08T02:23:00,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/c18d6771f6d346ab9eae05977b98be18 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/c18d6771f6d346ab9eae05977b98be18 2024-11-08T02:23:00,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e7f9f5bfeeff435c9c6547c88ab77b25 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e7f9f5bfeeff435c9c6547c88ab77b25 2024-11-08T02:23:00,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/b68a0c4e8c8848958a78ba40253dcab6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/b68a0c4e8c8848958a78ba40253dcab6 2024-11-08T02:23:00,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9d5d60c5dabe4f7eafd20209c9aece61 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/9d5d60c5dabe4f7eafd20209c9aece61 2024-11-08T02:23:00,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/693403b20e9f4dfdb01e19b4b325c72b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/693403b20e9f4dfdb01e19b4b325c72b 2024-11-08T02:23:00,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e509d4d02fd641d59dab07e87a2c57a5 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/e509d4d02fd641d59dab07e87a2c57a5 2024-11-08T02:23:00,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d77a6eb4aa7d40fd88aa185fa27a5fc6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/d77a6eb4aa7d40fd88aa185fa27a5fc6 2024-11-08T02:23:00,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/0655523e4be747ddbba60caaa903c634 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/0655523e4be747ddbba60caaa903c634 2024-11-08T02:23:00,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/b813115baa9e4451b55cde7f2fb1a115 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/b813115baa9e4451b55cde7f2fb1a115 2024-11-08T02:23:00,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/205b5812c98b4d3d92f5b2991bcd8d6f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/037afbce69b54f579c40f09adccd730b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d0ebfd02ef4e4c19a8d95f412097171a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec9d773296334b34b887917c324006ea, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d960a4b6a2fe46928b55b1ff03f8f533, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/30f830fe10b84282817a3fc8e76ef9e6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/590f4f45faa6453688a5ed34b4b9a51b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/01c10b8616bf434bb304a275b23ed402, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/0b02ed2c55df4909a26aae7bffe59dfa, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d268b491df044fbb9d7f3424ff407384, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/718201a4bd104070b5ab690fb0792cd0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e15195eb7b4f4580b25ad3509013356a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/92d78fac33b847a0876c59f004bc72cd, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a1562ac02c6e4041b6246e9cbf743d7e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/73165eb203af4b8384a4df451c34f104, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4e29c301486f4473a63d8aae55f6b65b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/503d9a0ad98f417e81e374eec1c40137, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4d84ad0bb7a849c3bcb17aa41038214a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/7feb6f150748491697fe149e00816733, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/093df2fd5ece4a6694df9c3471bc2407, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec38824da9c94930b67d515e29b2abde, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/999d2b88e65c4fa89301518bfd351b0d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a32016a868f24dc4b13c5e6d751360b0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/24b1f3ee55dc46459f6c4b184edbe03c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/7dac066ea7744c279819070f68d41ff8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/dca815b2b8764746bdac1c7ad70173a4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/11323b1b82184811b0a30e9c23f748cc, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ea344d37d002489c829b0539dc0457c4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/c1b73714d77e4ca1bbd2d7259c985d5b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/215accbf467b435598c44da7d3225b21, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e3780d1ed70540a9baec08063ff41658, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/0a36b5c25c044f6185207f460b8833ed, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/5b55ac1ae38d4ed1bcb01c70b88b6207, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ce3cbc6bb9f747b48075ba7db4b86f04, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/23dabb1a215b4009b4c0a2f8f18e4283, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/58e482fb2c8140be8a9b72110b45bf99, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/845e04505a554b3a859b73380f52d5ed, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e8d0922176a9429ca43c2d4949fa13ae] to archive 2024-11-08T02:23:00,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:23:00,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/205b5812c98b4d3d92f5b2991bcd8d6f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/205b5812c98b4d3d92f5b2991bcd8d6f 2024-11-08T02:23:00,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/037afbce69b54f579c40f09adccd730b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/037afbce69b54f579c40f09adccd730b 2024-11-08T02:23:00,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d0ebfd02ef4e4c19a8d95f412097171a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d0ebfd02ef4e4c19a8d95f412097171a 2024-11-08T02:23:00,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec9d773296334b34b887917c324006ea to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec9d773296334b34b887917c324006ea 2024-11-08T02:23:00,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d960a4b6a2fe46928b55b1ff03f8f533 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d960a4b6a2fe46928b55b1ff03f8f533 2024-11-08T02:23:00,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/30f830fe10b84282817a3fc8e76ef9e6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/30f830fe10b84282817a3fc8e76ef9e6 2024-11-08T02:23:00,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/590f4f45faa6453688a5ed34b4b9a51b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/590f4f45faa6453688a5ed34b4b9a51b 2024-11-08T02:23:00,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/01c10b8616bf434bb304a275b23ed402 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/01c10b8616bf434bb304a275b23ed402 2024-11-08T02:23:00,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/0b02ed2c55df4909a26aae7bffe59dfa to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/0b02ed2c55df4909a26aae7bffe59dfa 2024-11-08T02:23:00,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d268b491df044fbb9d7f3424ff407384 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/d268b491df044fbb9d7f3424ff407384 2024-11-08T02:23:00,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/718201a4bd104070b5ab690fb0792cd0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/718201a4bd104070b5ab690fb0792cd0 2024-11-08T02:23:00,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e15195eb7b4f4580b25ad3509013356a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e15195eb7b4f4580b25ad3509013356a 2024-11-08T02:23:00,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/92d78fac33b847a0876c59f004bc72cd to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/92d78fac33b847a0876c59f004bc72cd 2024-11-08T02:23:00,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a1562ac02c6e4041b6246e9cbf743d7e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a1562ac02c6e4041b6246e9cbf743d7e 2024-11-08T02:23:00,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/73165eb203af4b8384a4df451c34f104 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/73165eb203af4b8384a4df451c34f104 2024-11-08T02:23:00,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4e29c301486f4473a63d8aae55f6b65b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4e29c301486f4473a63d8aae55f6b65b 2024-11-08T02:23:00,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/503d9a0ad98f417e81e374eec1c40137 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/503d9a0ad98f417e81e374eec1c40137 2024-11-08T02:23:00,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4d84ad0bb7a849c3bcb17aa41038214a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4d84ad0bb7a849c3bcb17aa41038214a 2024-11-08T02:23:00,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/7feb6f150748491697fe149e00816733 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/7feb6f150748491697fe149e00816733 2024-11-08T02:23:00,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/093df2fd5ece4a6694df9c3471bc2407 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/093df2fd5ece4a6694df9c3471bc2407 2024-11-08T02:23:00,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec38824da9c94930b67d515e29b2abde to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ec38824da9c94930b67d515e29b2abde 2024-11-08T02:23:00,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/999d2b88e65c4fa89301518bfd351b0d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/999d2b88e65c4fa89301518bfd351b0d 2024-11-08T02:23:00,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a32016a868f24dc4b13c5e6d751360b0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/a32016a868f24dc4b13c5e6d751360b0 2024-11-08T02:23:00,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/24b1f3ee55dc46459f6c4b184edbe03c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/24b1f3ee55dc46459f6c4b184edbe03c 2024-11-08T02:23:00,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/7dac066ea7744c279819070f68d41ff8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/7dac066ea7744c279819070f68d41ff8 2024-11-08T02:23:00,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/dca815b2b8764746bdac1c7ad70173a4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/dca815b2b8764746bdac1c7ad70173a4 2024-11-08T02:23:00,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/11323b1b82184811b0a30e9c23f748cc to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/11323b1b82184811b0a30e9c23f748cc 2024-11-08T02:23:00,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ea344d37d002489c829b0539dc0457c4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ea344d37d002489c829b0539dc0457c4 2024-11-08T02:23:00,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/c1b73714d77e4ca1bbd2d7259c985d5b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/c1b73714d77e4ca1bbd2d7259c985d5b 2024-11-08T02:23:00,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/215accbf467b435598c44da7d3225b21 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/215accbf467b435598c44da7d3225b21 2024-11-08T02:23:00,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e3780d1ed70540a9baec08063ff41658 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e3780d1ed70540a9baec08063ff41658 2024-11-08T02:23:00,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/0a36b5c25c044f6185207f460b8833ed to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/0a36b5c25c044f6185207f460b8833ed 2024-11-08T02:23:00,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/5b55ac1ae38d4ed1bcb01c70b88b6207 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/5b55ac1ae38d4ed1bcb01c70b88b6207 2024-11-08T02:23:00,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ce3cbc6bb9f747b48075ba7db4b86f04 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/ce3cbc6bb9f747b48075ba7db4b86f04 2024-11-08T02:23:00,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/23dabb1a215b4009b4c0a2f8f18e4283 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/23dabb1a215b4009b4c0a2f8f18e4283 2024-11-08T02:23:00,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/58e482fb2c8140be8a9b72110b45bf99 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/58e482fb2c8140be8a9b72110b45bf99 2024-11-08T02:23:00,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/845e04505a554b3a859b73380f52d5ed to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/845e04505a554b3a859b73380f52d5ed 2024-11-08T02:23:00,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e8d0922176a9429ca43c2d4949fa13ae to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/e8d0922176a9429ca43c2d4949fa13ae 2024-11-08T02:23:00,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/51670cba9b9b44c596d59e1551042eb5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b61047686df94aefbe008073647d86e1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/2875c5e6db024005b8b5788f0ece11f1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/22346e163c884a8e8f3e888939c4dfe8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/56ff89a8deee4ac0a8b492a230e85ab5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/445b3032a2744dc4bd11791dc441a8be, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/6b4aabedcce94c40b85b1fe3e76415ad, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/0434953c2e18421fa117f4982ba7cf46, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/e9df5b925325499bb8c145f3b6f9a96b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/1f9ecb8aa63941b6866c644d6714fb16, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/86e4eaed86664f84a19611cddaf82246, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/20a3ae5a19464281b9a47c40f38f01f8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/5bbf273b7c024b57b473f5757f593fca, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/263be9a44c7649ec95dc0a717ff26c98, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/ee20f4905e9849a19c0a5b3fd88a95df, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/596fc7a345ef4b2ba5d33d1977f63b4e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/3e3fe53113f34a759ec92812ce6889f6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/7e353b14a14b4f2e8a3f83b9135aa9ef, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/9d5340cc08c44d71b251eb2d4a23ddcb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a2e9d0bc245c44209bc0933d8d93c5bd, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/cbc2fd3ac04840e1a0a02f95645670b9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/34c130b0f1534dae8ba46a7345cf5e5c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b7fab156b04a406b8e0763819534155b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/adef1a219d004c4896957c17fea01c4e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/0dcaddcf39c04ec89572145a47f7e555, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/dfe569f3b01d4738a2e2c235ac2a8afe, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/d13fb045ad3142e993f3b7b3eb344752, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/54c0d16c9acd43829151b59e275aec80, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/de6694058e2b4026a544752361437593, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/e173a800cb014ac1a47db1f490c5db4f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/044f3ba17a194a56a25ee4826cb69953, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a8c922434d8346ddb04bb6f1797ab3e9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/9f08d6c003194636bbf8c60846a7595a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/fda6eb80b08b431bbe4cfb662943aa86, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/8cbdab21d4534b82bb7141025fee63bb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/077d8ffe2b4241958db48df40f544f00, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/928918b4ed134b85a80dc602e82838c4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b1ac23c5878f465c920b8e637152f42d] to archive 2024-11-08T02:23:00,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:23:00,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/51670cba9b9b44c596d59e1551042eb5 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/51670cba9b9b44c596d59e1551042eb5 2024-11-08T02:23:00,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b61047686df94aefbe008073647d86e1 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b61047686df94aefbe008073647d86e1 2024-11-08T02:23:00,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/2875c5e6db024005b8b5788f0ece11f1 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/2875c5e6db024005b8b5788f0ece11f1 2024-11-08T02:23:00,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/22346e163c884a8e8f3e888939c4dfe8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/22346e163c884a8e8f3e888939c4dfe8 2024-11-08T02:23:00,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/56ff89a8deee4ac0a8b492a230e85ab5 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/56ff89a8deee4ac0a8b492a230e85ab5 2024-11-08T02:23:00,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/445b3032a2744dc4bd11791dc441a8be to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/445b3032a2744dc4bd11791dc441a8be 2024-11-08T02:23:00,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/6b4aabedcce94c40b85b1fe3e76415ad to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/6b4aabedcce94c40b85b1fe3e76415ad 2024-11-08T02:23:00,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/0434953c2e18421fa117f4982ba7cf46 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/0434953c2e18421fa117f4982ba7cf46 2024-11-08T02:23:00,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/e9df5b925325499bb8c145f3b6f9a96b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/e9df5b925325499bb8c145f3b6f9a96b 2024-11-08T02:23:00,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/1f9ecb8aa63941b6866c644d6714fb16 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/1f9ecb8aa63941b6866c644d6714fb16 2024-11-08T02:23:00,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/86e4eaed86664f84a19611cddaf82246 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/86e4eaed86664f84a19611cddaf82246 2024-11-08T02:23:00,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/20a3ae5a19464281b9a47c40f38f01f8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/20a3ae5a19464281b9a47c40f38f01f8 2024-11-08T02:23:00,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/5bbf273b7c024b57b473f5757f593fca to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/5bbf273b7c024b57b473f5757f593fca 2024-11-08T02:23:00,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/263be9a44c7649ec95dc0a717ff26c98 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/263be9a44c7649ec95dc0a717ff26c98 2024-11-08T02:23:00,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/ee20f4905e9849a19c0a5b3fd88a95df to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/ee20f4905e9849a19c0a5b3fd88a95df 2024-11-08T02:23:00,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/596fc7a345ef4b2ba5d33d1977f63b4e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/596fc7a345ef4b2ba5d33d1977f63b4e 2024-11-08T02:23:00,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/3e3fe53113f34a759ec92812ce6889f6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/3e3fe53113f34a759ec92812ce6889f6 2024-11-08T02:23:00,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/7e353b14a14b4f2e8a3f83b9135aa9ef to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/7e353b14a14b4f2e8a3f83b9135aa9ef 2024-11-08T02:23:00,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/9d5340cc08c44d71b251eb2d4a23ddcb to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/9d5340cc08c44d71b251eb2d4a23ddcb 2024-11-08T02:23:00,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a2e9d0bc245c44209bc0933d8d93c5bd to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a2e9d0bc245c44209bc0933d8d93c5bd 2024-11-08T02:23:00,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/cbc2fd3ac04840e1a0a02f95645670b9 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/cbc2fd3ac04840e1a0a02f95645670b9 2024-11-08T02:23:00,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/34c130b0f1534dae8ba46a7345cf5e5c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/34c130b0f1534dae8ba46a7345cf5e5c 2024-11-08T02:23:00,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b7fab156b04a406b8e0763819534155b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b7fab156b04a406b8e0763819534155b 2024-11-08T02:23:00,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/adef1a219d004c4896957c17fea01c4e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/adef1a219d004c4896957c17fea01c4e 2024-11-08T02:23:00,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/0dcaddcf39c04ec89572145a47f7e555 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/0dcaddcf39c04ec89572145a47f7e555 2024-11-08T02:23:00,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/dfe569f3b01d4738a2e2c235ac2a8afe to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/dfe569f3b01d4738a2e2c235ac2a8afe 2024-11-08T02:23:00,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/d13fb045ad3142e993f3b7b3eb344752 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/d13fb045ad3142e993f3b7b3eb344752 2024-11-08T02:23:00,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/54c0d16c9acd43829151b59e275aec80 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/54c0d16c9acd43829151b59e275aec80 2024-11-08T02:23:00,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/de6694058e2b4026a544752361437593 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/de6694058e2b4026a544752361437593 2024-11-08T02:23:00,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/e173a800cb014ac1a47db1f490c5db4f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/e173a800cb014ac1a47db1f490c5db4f 2024-11-08T02:23:00,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/044f3ba17a194a56a25ee4826cb69953 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/044f3ba17a194a56a25ee4826cb69953 2024-11-08T02:23:00,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a8c922434d8346ddb04bb6f1797ab3e9 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a8c922434d8346ddb04bb6f1797ab3e9 2024-11-08T02:23:00,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/9f08d6c003194636bbf8c60846a7595a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/9f08d6c003194636bbf8c60846a7595a 2024-11-08T02:23:00,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/fda6eb80b08b431bbe4cfb662943aa86 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/fda6eb80b08b431bbe4cfb662943aa86 2024-11-08T02:23:00,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/8cbdab21d4534b82bb7141025fee63bb to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/8cbdab21d4534b82bb7141025fee63bb 2024-11-08T02:23:00,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/077d8ffe2b4241958db48df40f544f00 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/077d8ffe2b4241958db48df40f544f00 2024-11-08T02:23:00,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/928918b4ed134b85a80dc602e82838c4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/928918b4ed134b85a80dc602e82838c4 2024-11-08T02:23:00,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b1ac23c5878f465c920b8e637152f42d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/b1ac23c5878f465c920b8e637152f42d 2024-11-08T02:23:00,179 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/recovered.edits/573.seqid, newMaxSeqId=573, maxSeqId=1 2024-11-08T02:23:00,180 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. 2024-11-08T02:23:00,180 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 8469534ae513a00a18db97d665b015cd: Waiting for close lock at 1731032578660Running coprocessor pre-close hooks at 1731032578660Disabling compacts and flushes for region at 1731032578660Disabling writes for close at 1731032578660Obtaining lock to block concurrent updates at 1731032578660Preparing flush snapshotting stores in 8469534ae513a00a18db97d665b015cd at 1731032578660Finished memstore snapshotting TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731032578661 (+1 ms)Flushing stores of TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd. at 1731032578661Flushing 8469534ae513a00a18db97d665b015cd/A: creating writer at 1731032578661Flushing 8469534ae513a00a18db97d665b015cd/A: appending metadata at 1731032578665 (+4 ms)Flushing 8469534ae513a00a18db97d665b015cd/A: closing flushed file at 1731032578665Flushing 8469534ae513a00a18db97d665b015cd/B: creating writer at 1731032579086 (+421 ms)Flushing 8469534ae513a00a18db97d665b015cd/B: appending metadata at 1731032579090 (+4 ms)Flushing 8469534ae513a00a18db97d665b015cd/B: closing flushed file at 1731032579090Flushing 8469534ae513a00a18db97d665b015cd/C: creating writer at 1731032579505 (+415 ms)Flushing 8469534ae513a00a18db97d665b015cd/C: appending metadata at 1731032579507 (+2 ms)Flushing 8469534ae513a00a18db97d665b015cd/C: closing flushed file at 1731032579507Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69a2f503: reopening flushed file at 1731032579917 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fbceeda: reopening flushed file at 1731032579924 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@702cabab: reopening flushed file at 1731032579934 (+10 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8469534ae513a00a18db97d665b015cd in 1282ms, sequenceid=570, compaction requested=true at 1731032579942 (+8 ms)Writing region close event to WAL at 1731032580173 (+231 ms)Running coprocessor post-close hooks at 1731032580180 (+7 ms)Closed at 1731032580180 2024-11-08T02:23:00,184 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 8469534ae513a00a18db97d665b015cd 2024-11-08T02:23:00,185 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=8469534ae513a00a18db97d665b015cd, regionState=CLOSED 2024-11-08T02:23:00,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:23:00,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-08T02:23:00,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 8469534ae513a00a18db97d665b015cd, server=331c7316141f,34325,1731032548147 in 1.6890 sec 2024-11-08T02:23:00,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-08T02:23:00,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8469534ae513a00a18db97d665b015cd, UNASSIGN in 1.7030 sec 2024-11-08T02:23:00,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-08T02:23:00,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7130 sec 2024-11-08T02:23:00,205 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032580204"}]},"ts":"1731032580204"} 2024-11-08T02:23:00,207 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:23:00,207 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:23:00,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7360 sec 2024-11-08T02:23:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T02:23:00,629 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:00,633 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:23:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:00,640 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T02:23:00,642 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:00,647 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd 2024-11-08T02:23:00,652 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/recovered.edits] 2024-11-08T02:23:00,656 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1ab1e3647aa041d8a367b3e928ff9f3b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/1ab1e3647aa041d8a367b3e928ff9f3b 2024-11-08T02:23:00,658 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/65210d00e424465f9a0284507ca7d837 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/65210d00e424465f9a0284507ca7d837 2024-11-08T02:23:00,659 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7144498580b54918bdf3373d9b90c275 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/7144498580b54918bdf3373d9b90c275 2024-11-08T02:23:00,661 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/ff7135ba2b1349869e1f57d0c1049625 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/A/ff7135ba2b1349869e1f57d0c1049625 2024-11-08T02:23:00,663 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/3f0bf9f5ab8246b8a314159ad5cc5bcc to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/3f0bf9f5ab8246b8a314159ad5cc5bcc 2024-11-08T02:23:00,665 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4ac224888450478d9902df6fad4fb142 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/4ac224888450478d9902df6fad4fb142 2024-11-08T02:23:00,667 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/9bd7b9efb6d9410bb3f1e23c797c21ed to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/9bd7b9efb6d9410bb3f1e23c797c21ed 2024-11-08T02:23:00,670 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/b27c1956066c4f5fa5e29a4c23202549 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/B/b27c1956066c4f5fa5e29a4c23202549 2024-11-08T02:23:00,673 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/5a337572870a47a393f8ca32144c364c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/5a337572870a47a393f8ca32144c364c 2024-11-08T02:23:00,675 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/75b2c370c6904b4c9a7b3d575551d72e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/75b2c370c6904b4c9a7b3d575551d72e 2024-11-08T02:23:00,676 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a2e934eb16ff467ea2d856df556d51f6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/a2e934eb16ff467ea2d856df556d51f6 2024-11-08T02:23:00,678 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/dff8ab9f4b3945e88e4de40ddf5ac27a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/C/dff8ab9f4b3945e88e4de40ddf5ac27a 2024-11-08T02:23:00,682 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/recovered.edits/573.seqid to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd/recovered.edits/573.seqid 2024-11-08T02:23:00,682 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/8469534ae513a00a18db97d665b015cd 2024-11-08T02:23:00,682 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:23:00,687 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:00,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-08T02:23:00,696 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:23:00,724 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:23:00,726 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:00,726 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:23:00,726 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731032580726"}]},"ts":"9223372036854775807"} 2024-11-08T02:23:00,731 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T02:23:00,731 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8469534ae513a00a18db97d665b015cd, NAME => 'TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:23:00,731 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:23:00,731 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731032580731"}]},"ts":"9223372036854775807"} 2024-11-08T02:23:00,734 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T02:23:00,735 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:00,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 102 msec 2024-11-08T02:23:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T02:23:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T02:23:00,749 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:00,758 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/331c7316141f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-403120256_22 at /127.0.0.1:49866 [Waiting for operation #322] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-548309475_22 at /127.0.0.1:32774 [Waiting for operation #318] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;331c7316141f:34325-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/331c7316141f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=550 (was 417) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=7059 (was 8294) 2024-11-08T02:23:00,767 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=550, ProcessCount=11, AvailableMemoryMB=7058 2024-11-08T02:23:00,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:23:00,769 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:23:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:00,771 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:23:00,771 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:00,771 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-08T02:23:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T02:23:00,772 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:23:00,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741963_1139 (size=959) 2024-11-08T02:23:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T02:23:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T02:23:01,180 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee 2024-11-08T02:23:01,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741964_1140 (size=53) 2024-11-08T02:23:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T02:23:01,587 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:23:01,587 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0ecb51078105f519ce4baa47714771b2, disabling compactions & flushes 2024-11-08T02:23:01,587 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:01,587 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:01,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. after waiting 0 ms 2024-11-08T02:23:01,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:01,588 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:01,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ecb51078105f519ce4baa47714771b2: Waiting for close lock at 1731032581587Disabling compacts and flushes for region at 1731032581587Disabling writes for close at 1731032581588 (+1 ms)Writing region close event to WAL at 1731032581588Closed at 1731032581588 2024-11-08T02:23:01,590 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:23:01,591 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731032581590"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731032581590"}]},"ts":"1731032581590"} 2024-11-08T02:23:01,594 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T02:23:01,596 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:23:01,597 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032581596"}]},"ts":"1731032581596"} 2024-11-08T02:23:01,600 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:23:01,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, ASSIGN}] 2024-11-08T02:23:01,602 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, ASSIGN 2024-11-08T02:23:01,603 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, ASSIGN; state=OFFLINE, location=331c7316141f,34325,1731032548147; forceNewPlan=false, retain=false 2024-11-08T02:23:01,754 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0ecb51078105f519ce4baa47714771b2, regionState=OPENING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:23:01,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, ASSIGN because future has completed 2024-11-08T02:23:01,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:23:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T02:23:01,919 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:01,919 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:23:01,920 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,920 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:23:01,920 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,920 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,922 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,925 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ecb51078105f519ce4baa47714771b2 columnFamilyName A 2024-11-08T02:23:01,925 DEBUG [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:01,926 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(327): Store=0ecb51078105f519ce4baa47714771b2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:23:01,926 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,928 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ecb51078105f519ce4baa47714771b2 columnFamilyName B 2024-11-08T02:23:01,928 DEBUG [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:01,928 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(327): Store=0ecb51078105f519ce4baa47714771b2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:23:01,928 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,930 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ecb51078105f519ce4baa47714771b2 columnFamilyName C 2024-11-08T02:23:01,930 DEBUG [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:01,930 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(327): Store=0ecb51078105f519ce4baa47714771b2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:23:01,931 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,931 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,932 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,933 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,933 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,933 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:23:01,934 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,936 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:23:01,936 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 0ecb51078105f519ce4baa47714771b2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60930300, jitterRate=-0.09206777811050415}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:23:01,936 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:01,937 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 0ecb51078105f519ce4baa47714771b2: Running coprocessor pre-open hook at 1731032581920Writing region info on filesystem at 1731032581920Initializing all the Stores at 1731032581922 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032581922Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032581922Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032581922Cleaning up temporary data from old regions at 1731032581933 (+11 ms)Running coprocessor post-open hooks at 1731032581936 (+3 ms)Region opened successfully at 1731032581937 (+1 ms) 2024-11-08T02:23:01,938 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., pid=30, masterSystemTime=1731032581909 2024-11-08T02:23:01,940 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:01,940 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:01,941 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0ecb51078105f519ce4baa47714771b2, regionState=OPEN, openSeqNum=2, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:23:01,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:23:01,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-08T02:23:01,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 in 187 msec 2024-11-08T02:23:01,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-08T02:23:01,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, ASSIGN in 345 msec 2024-11-08T02:23:01,949 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:23:01,949 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032581949"}]},"ts":"1731032581949"} 2024-11-08T02:23:01,951 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:23:01,952 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:23:01,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-08T02:23:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-08T02:23:02,920 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:02,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:02,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:02,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:02,928 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4caf3862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:02,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:02,932 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:02,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c8a99a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:02,933 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:02,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:02,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:02,937 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T02:23:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 331c7316141f,41605,1731032547397 2024-11-08T02:23:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2dd3874a 2024-11-08T02:23:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:23:02,939 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:23:02,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:23:02,944 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:23:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:02,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741965_1141 (size=995) 2024-11-08T02:23:03,365 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T02:23:03,365 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T02:23:03,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:23:03,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, REOPEN/MOVE}] 2024-11-08T02:23:03,379 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, REOPEN/MOVE 2024-11-08T02:23:03,380 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0ecb51078105f519ce4baa47714771b2, regionState=CLOSING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:23:03,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, REOPEN/MOVE because future has completed 2024-11-08T02:23:03,382 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:23:03,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:23:03,536 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,536 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T02:23:03,536 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 0ecb51078105f519ce4baa47714771b2, disabling compactions & flushes 2024-11-08T02:23:03,536 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:03,536 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:03,536 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. after waiting 0 ms 2024-11-08T02:23:03,536 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:03,544 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T02:23:03,545 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:03,546 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 0ecb51078105f519ce4baa47714771b2: Waiting for close lock at 1731032583536Running coprocessor pre-close hooks at 1731032583536Disabling compacts and flushes for region at 1731032583536Disabling writes for close at 1731032583536Writing region close event to WAL at 1731032583537 (+1 ms)Running coprocessor post-close hooks at 1731032583545 (+8 ms)Closed at 1731032583545 2024-11-08T02:23:03,546 WARN [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 0ecb51078105f519ce4baa47714771b2 to self. 2024-11-08T02:23:03,549 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0ecb51078105f519ce4baa47714771b2, regionState=CLOSED 2024-11-08T02:23:03,550 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:23:03,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-08T02:23:03,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 in 170 msec 2024-11-08T02:23:03,554 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, REOPEN/MOVE; state=CLOSED, location=331c7316141f,34325,1731032548147; forceNewPlan=false, retain=true 2024-11-08T02:23:03,705 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0ecb51078105f519ce4baa47714771b2, regionState=OPENING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:23:03,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, REOPEN/MOVE because future has completed 2024-11-08T02:23:03,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:23:03,871 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:03,872 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:23:03,872 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,872 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:23:03,873 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,873 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,877 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,882 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ecb51078105f519ce4baa47714771b2 columnFamilyName A 2024-11-08T02:23:03,884 DEBUG [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:03,884 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(327): Store=0ecb51078105f519ce4baa47714771b2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:23:03,885 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,886 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ecb51078105f519ce4baa47714771b2 columnFamilyName B 2024-11-08T02:23:03,886 DEBUG [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:03,886 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(327): Store=0ecb51078105f519ce4baa47714771b2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:23:03,886 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,887 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ecb51078105f519ce4baa47714771b2 columnFamilyName C 2024-11-08T02:23:03,887 DEBUG [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:03,887 INFO [StoreOpener-0ecb51078105f519ce4baa47714771b2-1 {}] regionserver.HStore(327): Store=0ecb51078105f519ce4baa47714771b2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:23:03,887 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,888 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,889 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,890 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,890 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,891 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:23:03,892 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,892 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 0ecb51078105f519ce4baa47714771b2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68645769, jitterRate=0.022901669144630432}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:23:03,893 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,894 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 0ecb51078105f519ce4baa47714771b2: Running coprocessor pre-open hook at 1731032583873Writing region info on filesystem at 1731032583873Initializing all the Stores at 1731032583874 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032583874Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032583875 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032583875Cleaning up temporary data from old regions at 1731032583890 (+15 ms)Running coprocessor post-open hooks at 1731032583893 (+3 ms)Region opened successfully at 1731032583894 (+1 ms) 2024-11-08T02:23:03,895 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., pid=35, masterSystemTime=1731032583863 2024-11-08T02:23:03,897 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:03,897 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:03,897 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0ecb51078105f519ce4baa47714771b2, regionState=OPEN, openSeqNum=5, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:23:03,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:23:03,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-08T02:23:03,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 in 191 msec 2024-11-08T02:23:03,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-08T02:23:03,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, REOPEN/MOVE in 523 msec 2024-11-08T02:23:03,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-08T02:23:03,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 535 msec 2024-11-08T02:23:03,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 962 msec 2024-11-08T02:23:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-08T02:23:03,913 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:03,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:03,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:03,914 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:03,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:03,914 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11181819, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:03,916 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a399522, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:03,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:03,918 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bbd0af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:03,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:03,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:03,919 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:03,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@232a89a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:03,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:03,922 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1768c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:03,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:03,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711503ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,924 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:03,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:03,925 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb91011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:03,926 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e7e41f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:03,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:03,928 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2473021c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:03,929 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3f4e3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:03,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:03,931 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:03,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:03,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2614ec2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:03,932 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b1a1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:03,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:03,934 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a2829d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:03,936 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:03,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f0ca5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:03,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:03,938 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b0988f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:03,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:03,939 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f044a27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:03,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:03,941 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:03,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:03,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:03,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec142a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:03,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:03,942 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9373c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:03,944 DEBUG [Thread-706 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:03,944 DEBUG [Thread-699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:03,944 DEBUG [Thread-708 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:03,944 DEBUG [Thread-712 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:03,944 DEBUG [Thread-697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:03,944 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:03,945 DEBUG [Thread-710 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:03,945 DEBUG [Thread-701 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:03,945 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-08T02:23:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:03,945 DEBUG [Thread-703 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:03,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:03,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:03,946 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T02:23:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:03,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:03,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:03,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:03,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:03,948 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:03,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:03,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:03,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:03,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:03,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:03,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:03,949 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:03,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:03,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:23:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:23:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:23:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:23:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:23:03,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:23:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:23:03,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:23:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:23:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:03,983 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:03,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T02:23:04,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a54fc624904446b9842c9572f33beeee_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032583975/Put/seqid=0 2024-11-08T02:23:04,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741966_1142 (size=12154) 2024-11-08T02:23:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T02:23:04,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032644058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032644062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032644062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032644066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032644058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-08T02:23:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032644178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032644179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032644179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032644188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032644188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-08T02:23:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T02:23:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032644388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032644388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032644388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-08T02:23:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032644408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032644408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:04,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:04,438 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a54fc624904446b9842c9572f33beeee_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a54fc624904446b9842c9572f33beeee_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:04,440 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/f82641e8ddb4492d88ec1c3ee91c9b44, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:04,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/f82641e8ddb4492d88ec1c3ee91c9b44 is 175, key is test_row_0/A:col10/1731032583975/Put/seqid=0 2024-11-08T02:23:04,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741967_1143 (size=30955) 2024-11-08T02:23:04,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-08T02:23:04,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:04,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T02:23:04,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032644698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032644708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032644708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032644718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032644718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:04,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-08T02:23:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:04,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,862 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/f82641e8ddb4492d88ec1c3ee91c9b44 2024-11-08T02:23:04,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-08T02:23:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:04,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:04,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/3c62cfd5a5bb4e068743f252d10b88be is 50, key is test_row_0/B:col10/1731032583975/Put/seqid=0 2024-11-08T02:23:04,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741968_1144 (size=12001) 2024-11-08T02:23:04,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/3c62cfd5a5bb4e068743f252d10b88be 2024-11-08T02:23:04,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/ff868e9a25bc430686bf59ea1e816cc7 is 50, key is test_row_0/C:col10/1731032583975/Put/seqid=0 2024-11-08T02:23:04,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741969_1145 (size=12001) 2024-11-08T02:23:04,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/ff868e9a25bc430686bf59ea1e816cc7 2024-11-08T02:23:04,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/f82641e8ddb4492d88ec1c3ee91c9b44 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f82641e8ddb4492d88ec1c3ee91c9b44 2024-11-08T02:23:04,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f82641e8ddb4492d88ec1c3ee91c9b44, entries=150, sequenceid=17, filesize=30.2 K 2024-11-08T02:23:04,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/3c62cfd5a5bb4e068743f252d10b88be as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/3c62cfd5a5bb4e068743f252d10b88be 2024-11-08T02:23:04,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/3c62cfd5a5bb4e068743f252d10b88be, entries=150, sequenceid=17, filesize=11.7 K 2024-11-08T02:23:04,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/ff868e9a25bc430686bf59ea1e816cc7 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ff868e9a25bc430686bf59ea1e816cc7 2024-11-08T02:23:04,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ff868e9a25bc430686bf59ea1e816cc7, entries=150, sequenceid=17, filesize=11.7 K 2024-11-08T02:23:04,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0ecb51078105f519ce4baa47714771b2 in 1006ms, sequenceid=17, compaction requested=false 2024-11-08T02:23:04,989 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T02:23:04,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:05,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-08T02:23:05,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:05,031 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T02:23:05,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089bdf34172da3430cb96c9f9922553c48_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032584055/Put/seqid=0 2024-11-08T02:23:05,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741970_1146 (size=12154) 2024-11-08T02:23:05,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:05,069 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089bdf34172da3430cb96c9f9922553c48_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089bdf34172da3430cb96c9f9922553c48_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:05,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/c97b19edbe4a4c808663563b493f1512, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:05,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/c97b19edbe4a4c808663563b493f1512 is 175, key is test_row_0/A:col10/1731032584055/Put/seqid=0 2024-11-08T02:23:05,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741971_1147 (size=30955) 2024-11-08T02:23:05,086 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/c97b19edbe4a4c808663563b493f1512 2024-11-08T02:23:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T02:23:05,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/fa4f5102a92447a3a85bede21032361b is 50, key is test_row_0/B:col10/1731032584055/Put/seqid=0 2024-11-08T02:23:05,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741972_1148 (size=12001) 2024-11-08T02:23:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:05,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:05,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032645231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032645232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032645234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032645235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032645238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032645338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032645348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032645349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032645349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,517 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/fa4f5102a92447a3a85bede21032361b 2024-11-08T02:23:05,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/b328bab3893a422bb452ca050f5c0d5d is 50, key is test_row_0/C:col10/1731032584055/Put/seqid=0 2024-11-08T02:23:05,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741973_1149 (size=12001) 2024-11-08T02:23:05,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032645548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032645559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032645559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032645559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,732 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:23:05,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032645858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032645869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032645869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032645869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:05,930 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/b328bab3893a422bb452ca050f5c0d5d 2024-11-08T02:23:05,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/c97b19edbe4a4c808663563b493f1512 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c97b19edbe4a4c808663563b493f1512 2024-11-08T02:23:05,949 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c97b19edbe4a4c808663563b493f1512, entries=150, sequenceid=40, filesize=30.2 K 2024-11-08T02:23:05,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/fa4f5102a92447a3a85bede21032361b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/fa4f5102a92447a3a85bede21032361b 2024-11-08T02:23:05,955 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/fa4f5102a92447a3a85bede21032361b, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:23:05,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/b328bab3893a422bb452ca050f5c0d5d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/b328bab3893a422bb452ca050f5c0d5d 2024-11-08T02:23:05,964 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/b328bab3893a422bb452ca050f5c0d5d, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:23:05,965 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 0ecb51078105f519ce4baa47714771b2 in 934ms, sequenceid=40, compaction requested=false 2024-11-08T02:23:05,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:05,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:05,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-08T02:23:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-08T02:23:05,973 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-08T02:23:05,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0220 sec 2024-11-08T02:23:05,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.0300 sec 2024-11-08T02:23:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-08T02:23:06,098 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:06,099 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-08T02:23:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T02:23:06,100 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:06,101 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:06,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T02:23:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:06,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:06,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T02:23:06,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-08T02:23:06,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:06,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fb5770b256b34f7bb40dd47c8787b4f0_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032586248/Put/seqid=0 2024-11-08T02:23:06,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741974_1150 (size=12154) 2024-11-08T02:23:06,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:06,282 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fb5770b256b34f7bb40dd47c8787b4f0_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fb5770b256b34f7bb40dd47c8787b4f0_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:06,283 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/84868687bd1340238b318d44bbf427c4, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:06,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/84868687bd1340238b318d44bbf427c4 is 175, key is test_row_0/A:col10/1731032586248/Put/seqid=0 2024-11-08T02:23:06,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741975_1151 (size=30955) 2024-11-08T02:23:06,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032646339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:06,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032646367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:06,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:06,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032646378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032646378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:06,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032646378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:06,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:06,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-08T02:23:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T02:23:06,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032646448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:06,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:06,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-08T02:23:06,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:06,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032646658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:06,694 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/84868687bd1340238b318d44bbf427c4 2024-11-08T02:23:06,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/bd99e9548c904895884c00f5b440d6f3 is 50, key is test_row_0/B:col10/1731032586248/Put/seqid=0 2024-11-08T02:23:06,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-08T02:23:06,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:06,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T02:23:06,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741976_1152 (size=12001) 2024-11-08T02:23:06,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/bd99e9548c904895884c00f5b440d6f3 2024-11-08T02:23:06,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/3431d6afdd1d45189fc389e29b27f760 is 50, key is test_row_0/C:col10/1731032586248/Put/seqid=0 2024-11-08T02:23:06,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741977_1153 (size=12001) 2024-11-08T02:23:06,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/3431d6afdd1d45189fc389e29b27f760 2024-11-08T02:23:06,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/84868687bd1340238b318d44bbf427c4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/84868687bd1340238b318d44bbf427c4 2024-11-08T02:23:06,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/84868687bd1340238b318d44bbf427c4, entries=150, sequenceid=54, filesize=30.2 K 2024-11-08T02:23:06,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/bd99e9548c904895884c00f5b440d6f3 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/bd99e9548c904895884c00f5b440d6f3 2024-11-08T02:23:06,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/bd99e9548c904895884c00f5b440d6f3, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T02:23:06,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/3431d6afdd1d45189fc389e29b27f760 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/3431d6afdd1d45189fc389e29b27f760 2024-11-08T02:23:06,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/3431d6afdd1d45189fc389e29b27f760, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T02:23:06,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 0ecb51078105f519ce4baa47714771b2 in 548ms, sequenceid=54, compaction requested=true 2024-11-08T02:23:06,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:06,798 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:06,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:06,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:06,798 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:06,799 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:06,799 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/A is initiating minor compaction (all files) 2024-11-08T02:23:06,799 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/A in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,800 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f82641e8ddb4492d88ec1c3ee91c9b44, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c97b19edbe4a4c808663563b493f1512, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/84868687bd1340238b318d44bbf427c4] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=90.7 K 2024-11-08T02:23:06,800 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,800 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f82641e8ddb4492d88ec1c3ee91c9b44, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c97b19edbe4a4c808663563b493f1512, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/84868687bd1340238b318d44bbf427c4] 2024-11-08T02:23:06,800 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:06,800 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/B is initiating minor compaction (all files) 2024-11-08T02:23:06,800 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/B in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,801 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/3c62cfd5a5bb4e068743f252d10b88be, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/fa4f5102a92447a3a85bede21032361b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/bd99e9548c904895884c00f5b440d6f3] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=35.2 K 2024-11-08T02:23:06,802 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c62cfd5a5bb4e068743f252d10b88be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731032583969 2024-11-08T02:23:06,802 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting f82641e8ddb4492d88ec1c3ee91c9b44, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731032583969 2024-11-08T02:23:06,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:06,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:06,803 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting fa4f5102a92447a3a85bede21032361b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731032584011 2024-11-08T02:23:06,803 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c97b19edbe4a4c808663563b493f1512, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731032584011 2024-11-08T02:23:06,803 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting bd99e9548c904895884c00f5b440d6f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032585224 2024-11-08T02:23:06,804 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84868687bd1340238b318d44bbf427c4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032585224 2024-11-08T02:23:06,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:06,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:06,831 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:06,832 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:06,832 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/3ddd441331d543938eb92ac6b212406d is 50, key is test_row_0/B:col10/1731032586248/Put/seqid=0 2024-11-08T02:23:06,846 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110899dc3a85e6224093a1b46fe043931792_0ecb51078105f519ce4baa47714771b2 store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:06,854 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110899dc3a85e6224093a1b46fe043931792_0ecb51078105f519ce4baa47714771b2, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:06,855 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110899dc3a85e6224093a1b46fe043931792_0ecb51078105f519ce4baa47714771b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:06,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-08T02:23:06,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:06,871 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T02:23:06,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741978_1154 (size=12104) 2024-11-08T02:23:06,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110814c236e72809427185721e5e47d8844a_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032586332/Put/seqid=0 2024-11-08T02:23:06,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741979_1155 (size=4469) 2024-11-08T02:23:06,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741980_1156 (size=12154) 2024-11-08T02:23:06,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:06,931 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110814c236e72809427185721e5e47d8844a_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110814c236e72809427185721e5e47d8844a_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:06,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/6fbcb5c3d641475aa13d9b393e7f220d, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:06,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/6fbcb5c3d641475aa13d9b393e7f220d is 175, key is test_row_0/A:col10/1731032586332/Put/seqid=0 2024-11-08T02:23:06,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741981_1157 (size=30955) 2024-11-08T02:23:06,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:07,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032647019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:07,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:07,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032647127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T02:23:07,281 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/3ddd441331d543938eb92ac6b212406d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/3ddd441331d543938eb92ac6b212406d 2024-11-08T02:23:07,287 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/B of 0ecb51078105f519ce4baa47714771b2 into 3ddd441331d543938eb92ac6b212406d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:07,287 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:07,287 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/B, priority=13, startTime=1731032586798; duration=0sec 2024-11-08T02:23:07,287 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:07,287 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:B 2024-11-08T02:23:07,287 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:07,289 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:07,289 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/C is initiating minor compaction (all files) 2024-11-08T02:23:07,289 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/C in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:07,289 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ff868e9a25bc430686bf59ea1e816cc7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/b328bab3893a422bb452ca050f5c0d5d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/3431d6afdd1d45189fc389e29b27f760] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=35.2 K 2024-11-08T02:23:07,290 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting ff868e9a25bc430686bf59ea1e816cc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731032583969 2024-11-08T02:23:07,290 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting b328bab3893a422bb452ca050f5c0d5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731032584011 2024-11-08T02:23:07,290 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 3431d6afdd1d45189fc389e29b27f760, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032585224 2024-11-08T02:23:07,300 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#C#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:07,300 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/dbfbbef33cf54c5497137e46d574f845 is 50, key is test_row_0/C:col10/1731032586248/Put/seqid=0 2024-11-08T02:23:07,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741982_1158 (size=12104) 2024-11-08T02:23:07,321 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#A#compaction#136 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:07,321 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/19f01ddaae60428fb09562252356d7a9 is 175, key is test_row_0/A:col10/1731032586248/Put/seqid=0 2024-11-08T02:23:07,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741983_1159 (size=31058) 2024-11-08T02:23:07,344 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/6fbcb5c3d641475aa13d9b393e7f220d 2024-11-08T02:23:07,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032647338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:07,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/6b8e791cf68844f1b0cda1b0f338fd3f is 50, key is test_row_0/B:col10/1731032586332/Put/seqid=0 2024-11-08T02:23:07,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741984_1160 (size=12001) 2024-11-08T02:23:07,370 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/6b8e791cf68844f1b0cda1b0f338fd3f 2024-11-08T02:23:07,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032647378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:07,382 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:07,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032647389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:07,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:07,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032647389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:07,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/d3e6e8499ec64766b758969d9a42a611 is 50, key is test_row_0/C:col10/1731032586332/Put/seqid=0 2024-11-08T02:23:07,393 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:07,392 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:07,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032647389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:07,396 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:07,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741985_1161 (size=12001) 2024-11-08T02:23:07,406 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/d3e6e8499ec64766b758969d9a42a611 2024-11-08T02:23:07,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/6fbcb5c3d641475aa13d9b393e7f220d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/6fbcb5c3d641475aa13d9b393e7f220d 2024-11-08T02:23:07,423 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/6fbcb5c3d641475aa13d9b393e7f220d, entries=150, sequenceid=76, filesize=30.2 K 2024-11-08T02:23:07,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/6b8e791cf68844f1b0cda1b0f338fd3f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6b8e791cf68844f1b0cda1b0f338fd3f 2024-11-08T02:23:07,431 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6b8e791cf68844f1b0cda1b0f338fd3f, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:23:07,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/d3e6e8499ec64766b758969d9a42a611 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d3e6e8499ec64766b758969d9a42a611 2024-11-08T02:23:07,440 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d3e6e8499ec64766b758969d9a42a611, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:23:07,441 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 0ecb51078105f519ce4baa47714771b2 in 570ms, sequenceid=76, compaction requested=false 2024-11-08T02:23:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:07,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-08T02:23:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-08T02:23:07,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-08T02:23:07,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3420 sec 2024-11-08T02:23:07,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.3480 sec 2024-11-08T02:23:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:07,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:07,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T02:23:07,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085b6ed291bbcb4ac4aac66b248c06684b_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032587008/Put/seqid=0 2024-11-08T02:23:07,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741986_1162 (size=14594) 2024-11-08T02:23:07,718 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/dbfbbef33cf54c5497137e46d574f845 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/dbfbbef33cf54c5497137e46d574f845 2024-11-08T02:23:07,730 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/C of 0ecb51078105f519ce4baa47714771b2 into dbfbbef33cf54c5497137e46d574f845(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:07,730 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:07,730 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/C, priority=13, startTime=1731032586802; duration=0sec 2024-11-08T02:23:07,730 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:07,730 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:C 2024-11-08T02:23:07,737 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/19f01ddaae60428fb09562252356d7a9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/19f01ddaae60428fb09562252356d7a9 2024-11-08T02:23:07,747 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/A of 0ecb51078105f519ce4baa47714771b2 into 19f01ddaae60428fb09562252356d7a9(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:07,748 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:07,748 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/A, priority=13, startTime=1731032586798; duration=0sec 2024-11-08T02:23:07,748 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:07,748 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:A 2024-11-08T02:23:07,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032647773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:07,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032647888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:08,099 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:08,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032648098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:08,105 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085b6ed291bbcb4ac4aac66b248c06684b_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085b6ed291bbcb4ac4aac66b248c06684b_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:08,106 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/65218c244c80494c889ae033e34c66d4, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:08,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/65218c244c80494c889ae033e34c66d4 is 175, key is test_row_0/A:col10/1731032587008/Put/seqid=0 2024-11-08T02:23:08,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741987_1163 (size=39549) 2024-11-08T02:23:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-08T02:23:08,249 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:08,249 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-08T02:23:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T02:23:08,252 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:08,255 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:08,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T02:23:08,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T02:23:08,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:08,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:08,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:08,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:08,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:08,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032648408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:08,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:08,513 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/65218c244c80494c889ae033e34c66d4 2024-11-08T02:23:08,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/e19552a2f3c24b3791c953cbd7bd38e8 is 50, key is test_row_0/B:col10/1731032587008/Put/seqid=0 2024-11-08T02:23:08,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741988_1164 (size=12001) 2024-11-08T02:23:08,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/e19552a2f3c24b3791c953cbd7bd38e8 2024-11-08T02:23:08,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/252831ea77bd41a5802dfb7a37c03aa4 is 50, key is test_row_0/C:col10/1731032587008/Put/seqid=0 2024-11-08T02:23:08,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T02:23:08,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:08,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:08,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:08,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:08,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T02:23:08,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741989_1165 (size=12001) 2024-11-08T02:23:08,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/252831ea77bd41a5802dfb7a37c03aa4 2024-11-08T02:23:08,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/65218c244c80494c889ae033e34c66d4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/65218c244c80494c889ae033e34c66d4 2024-11-08T02:23:08,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/65218c244c80494c889ae033e34c66d4, entries=200, sequenceid=92, filesize=38.6 K 2024-11-08T02:23:08,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/e19552a2f3c24b3791c953cbd7bd38e8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e19552a2f3c24b3791c953cbd7bd38e8 2024-11-08T02:23:08,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e19552a2f3c24b3791c953cbd7bd38e8, entries=150, sequenceid=92, filesize=11.7 K 2024-11-08T02:23:08,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/252831ea77bd41a5802dfb7a37c03aa4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/252831ea77bd41a5802dfb7a37c03aa4 2024-11-08T02:23:08,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/252831ea77bd41a5802dfb7a37c03aa4, entries=150, sequenceid=92, filesize=11.7 K 2024-11-08T02:23:08,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0ecb51078105f519ce4baa47714771b2 in 946ms, sequenceid=92, compaction requested=true 2024-11-08T02:23:08,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:08,607 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:08,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:08,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:08,608 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:08,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:08,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:08,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:08,608 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:08,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:08,608 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/A is initiating minor compaction (all files) 2024-11-08T02:23:08,609 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:08,609 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/A in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:08,609 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/B is initiating minor compaction (all files) 2024-11-08T02:23:08,609 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/B in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:08,609 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/19f01ddaae60428fb09562252356d7a9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/6fbcb5c3d641475aa13d9b393e7f220d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/65218c244c80494c889ae033e34c66d4] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=99.2 K 2024-11-08T02:23:08,609 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:08,609 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/3ddd441331d543938eb92ac6b212406d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6b8e791cf68844f1b0cda1b0f338fd3f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e19552a2f3c24b3791c953cbd7bd38e8] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=35.3 K 2024-11-08T02:23:08,609 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/19f01ddaae60428fb09562252356d7a9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/6fbcb5c3d641475aa13d9b393e7f220d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/65218c244c80494c889ae033e34c66d4] 2024-11-08T02:23:08,609 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ddd441331d543938eb92ac6b212406d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032585224 2024-11-08T02:23:08,609 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19f01ddaae60428fb09562252356d7a9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032585224 2024-11-08T02:23:08,609 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fbcb5c3d641475aa13d9b393e7f220d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731032586325 2024-11-08T02:23:08,609 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b8e791cf68844f1b0cda1b0f338fd3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731032586325 2024-11-08T02:23:08,610 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65218c244c80494c889ae033e34c66d4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731032586974 2024-11-08T02:23:08,610 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting e19552a2f3c24b3791c953cbd7bd38e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731032587008 2024-11-08T02:23:08,621 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:08,622 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#B#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:08,623 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/1965fc4db1c64016acd9043ec5479364 is 50, key is test_row_0/B:col10/1731032587008/Put/seqid=0 2024-11-08T02:23:08,625 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108f862c419ab1a435ba7c142e37603449c_0ecb51078105f519ce4baa47714771b2 store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:08,629 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108f862c419ab1a435ba7c142e37603449c_0ecb51078105f519ce4baa47714771b2, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:08,629 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f862c419ab1a435ba7c142e37603449c_0ecb51078105f519ce4baa47714771b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:08,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741990_1166 (size=12207) 2024-11-08T02:23:08,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741991_1167 (size=4469) 2024-11-08T02:23:08,659 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#A#compaction#145 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:08,659 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/0257d2c641c54c4ea218ca76ada5ba03 is 175, key is test_row_0/A:col10/1731032587008/Put/seqid=0 2024-11-08T02:23:08,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741992_1168 (size=31161) 2024-11-08T02:23:08,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T02:23:08,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:08,715 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T02:23:08,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108693449eea26c43fdab5abaa6bd310fea_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032587770/Put/seqid=0 2024-11-08T02:23:08,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741993_1169 (size=12154) 2024-11-08T02:23:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T02:23:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:08,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:08,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032648962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:09,045 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/1965fc4db1c64016acd9043ec5479364 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1965fc4db1c64016acd9043ec5479364 2024-11-08T02:23:09,051 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/B of 0ecb51078105f519ce4baa47714771b2 into 1965fc4db1c64016acd9043ec5479364(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:09,051 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:09,051 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/B, priority=13, startTime=1731032588607; duration=0sec 2024-11-08T02:23:09,051 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:09,051 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:B 2024-11-08T02:23:09,051 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:09,053 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:09,053 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/C is initiating minor compaction (all files) 2024-11-08T02:23:09,053 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/C in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:09,053 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/dbfbbef33cf54c5497137e46d574f845, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d3e6e8499ec64766b758969d9a42a611, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/252831ea77bd41a5802dfb7a37c03aa4] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=35.3 K 2024-11-08T02:23:09,053 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting dbfbbef33cf54c5497137e46d574f845, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731032585224 2024-11-08T02:23:09,054 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting d3e6e8499ec64766b758969d9a42a611, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731032586325 2024-11-08T02:23:09,057 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 252831ea77bd41a5802dfb7a37c03aa4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731032587008 2024-11-08T02:23:09,065 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:09,065 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/4d5495af73b545fb887ae37459244334 is 50, key is test_row_0/C:col10/1731032587008/Put/seqid=0 2024-11-08T02:23:09,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741994_1170 (size=12207) 2024-11-08T02:23:09,075 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/0257d2c641c54c4ea218ca76ada5ba03 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/0257d2c641c54c4ea218ca76ada5ba03 2024-11-08T02:23:09,080 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/A of 0ecb51078105f519ce4baa47714771b2 into 0257d2c641c54c4ea218ca76ada5ba03(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:09,080 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:09,081 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/A, priority=13, startTime=1731032588607; duration=0sec 2024-11-08T02:23:09,081 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:09,081 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:A 2024-11-08T02:23:09,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032649078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:09,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:09,132 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108693449eea26c43fdab5abaa6bd310fea_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108693449eea26c43fdab5abaa6bd310fea_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:09,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/527148a7b7ed435b975de29dcc3599a6, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:09,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/527148a7b7ed435b975de29dcc3599a6 is 175, key is test_row_0/A:col10/1731032587770/Put/seqid=0 2024-11-08T02:23:09,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741995_1171 (size=30955) 2024-11-08T02:23:09,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032649288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:09,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T02:23:09,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032649398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:09,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032649398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:09,400 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,401 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:09,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032649408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:09,410 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:09,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032649417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:09,420 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:09,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:09,476 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/4d5495af73b545fb887ae37459244334 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/4d5495af73b545fb887ae37459244334 2024-11-08T02:23:09,484 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/C of 0ecb51078105f519ce4baa47714771b2 into 4d5495af73b545fb887ae37459244334(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:09,484 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:09,484 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/C, priority=13, startTime=1731032588608; duration=0sec 2024-11-08T02:23:09,486 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:09,486 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:C 2024-11-08T02:23:09,539 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/527148a7b7ed435b975de29dcc3599a6 2024-11-08T02:23:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/515297aff9f24da8859fdaa7498379ec is 50, key is test_row_0/B:col10/1731032587770/Put/seqid=0 2024-11-08T02:23:09,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741996_1172 (size=12001) 2024-11-08T02:23:09,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032649598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:09,955 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/515297aff9f24da8859fdaa7498379ec 2024-11-08T02:23:09,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/ea77ad4937874145af048171939db707 is 50, key is test_row_0/C:col10/1731032587770/Put/seqid=0 2024-11-08T02:23:09,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741997_1173 (size=12001) 2024-11-08T02:23:09,969 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/ea77ad4937874145af048171939db707 2024-11-08T02:23:09,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/527148a7b7ed435b975de29dcc3599a6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/527148a7b7ed435b975de29dcc3599a6 2024-11-08T02:23:09,978 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/527148a7b7ed435b975de29dcc3599a6, entries=150, sequenceid=115, filesize=30.2 K 2024-11-08T02:23:09,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/515297aff9f24da8859fdaa7498379ec as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/515297aff9f24da8859fdaa7498379ec 2024-11-08T02:23:09,985 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/515297aff9f24da8859fdaa7498379ec, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:23:09,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/ea77ad4937874145af048171939db707 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ea77ad4937874145af048171939db707 2024-11-08T02:23:09,996 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ea77ad4937874145af048171939db707, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:23:09,999 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 0ecb51078105f519ce4baa47714771b2 in 1284ms, sequenceid=115, compaction requested=false 2024-11-08T02:23:09,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:09,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:09,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-08T02:23:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-08T02:23:10,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-08T02:23:10,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7450 sec 2024-11-08T02:23:10,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.7550 sec 2024-11-08T02:23:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:10,110 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:10,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T02:23:10,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108049b4896bc514e41ab2f0b45e48c2476_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032590109/Put/seqid=0 2024-11-08T02:23:10,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741998_1174 (size=12254) 2024-11-08T02:23:10,130 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:10,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108049b4896bc514e41ab2f0b45e48c2476_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108049b4896bc514e41ab2f0b45e48c2476_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:10,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/5fb98af47044414598ef421762e922d1, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:10,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/5fb98af47044414598ef421762e922d1 is 175, key is test_row_0/A:col10/1731032590109/Put/seqid=0 2024-11-08T02:23:10,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741999_1175 (size=31055) 2024-11-08T02:23:10,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032650175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:10,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032650288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T02:23:10,398 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:10,399 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-08T02:23:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T02:23:10,400 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:10,401 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:10,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:10,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032650498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:10,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T02:23:10,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/5fb98af47044414598ef421762e922d1 2024-11-08T02:23:10,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T02:23:10,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:10,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:10,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:10,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:10,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:10,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/2200a056b078476587df54443bbf9c46 is 50, key is test_row_0/B:col10/1731032590109/Put/seqid=0 2024-11-08T02:23:10,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742000_1176 (size=12101) 2024-11-08T02:23:10,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T02:23:10,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:10,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:10,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:10,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:10,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T02:23:10,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032650808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:10,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T02:23:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:10,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:10,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:10,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/2200a056b078476587df54443bbf9c46 2024-11-08T02:23:11,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T02:23:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:11,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:11,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:11,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/07e246e21c0e4478bb5a326c032e7928 is 50, key is test_row_0/C:col10/1731032590109/Put/seqid=0 2024-11-08T02:23:11,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742001_1177 (size=12101) 2024-11-08T02:23:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T02:23:11,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/07e246e21c0e4478bb5a326c032e7928 2024-11-08T02:23:11,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/5fb98af47044414598ef421762e922d1 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5fb98af47044414598ef421762e922d1 2024-11-08T02:23:11,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5fb98af47044414598ef421762e922d1, entries=150, sequenceid=134, filesize=30.3 K 2024-11-08T02:23:11,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/2200a056b078476587df54443bbf9c46 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/2200a056b078476587df54443bbf9c46 2024-11-08T02:23:11,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/2200a056b078476587df54443bbf9c46, entries=150, sequenceid=134, filesize=11.8 K 2024-11-08T02:23:11,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/07e246e21c0e4478bb5a326c032e7928 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/07e246e21c0e4478bb5a326c032e7928 2024-11-08T02:23:11,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/07e246e21c0e4478bb5a326c032e7928, entries=150, sequenceid=134, filesize=11.8 K 2024-11-08T02:23:11,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 0ecb51078105f519ce4baa47714771b2 in 938ms, sequenceid=134, compaction requested=true 2024-11-08T02:23:11,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:11,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:11,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:11,049 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:11,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:11,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:11,049 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:11,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:11,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:11,050 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:11,050 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:11,050 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/B is initiating minor compaction (all files) 2024-11-08T02:23:11,050 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/A is initiating minor compaction (all files) 2024-11-08T02:23:11,050 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/A in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:11,050 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/0257d2c641c54c4ea218ca76ada5ba03, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/527148a7b7ed435b975de29dcc3599a6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5fb98af47044414598ef421762e922d1] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=91.0 K 2024-11-08T02:23:11,050 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:11,050 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/0257d2c641c54c4ea218ca76ada5ba03, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/527148a7b7ed435b975de29dcc3599a6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5fb98af47044414598ef421762e922d1] 2024-11-08T02:23:11,050 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/B in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:11,050 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1965fc4db1c64016acd9043ec5479364, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/515297aff9f24da8859fdaa7498379ec, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/2200a056b078476587df54443bbf9c46] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=35.5 K 2024-11-08T02:23:11,051 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0257d2c641c54c4ea218ca76ada5ba03, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731032587008 2024-11-08T02:23:11,051 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 1965fc4db1c64016acd9043ec5479364, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731032587008 2024-11-08T02:23:11,051 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 527148a7b7ed435b975de29dcc3599a6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731032587732 2024-11-08T02:23:11,051 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 515297aff9f24da8859fdaa7498379ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731032587732 2024-11-08T02:23:11,051 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fb98af47044414598ef421762e922d1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731032588927 2024-11-08T02:23:11,051 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 2200a056b078476587df54443bbf9c46, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731032588927 2024-11-08T02:23:11,058 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:11,060 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411087ed72818f4554b0f8b735e6865d92c5e_0ecb51078105f519ce4baa47714771b2 store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:11,060 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:11,061 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/47f20814eb85463789488526eec993df is 50, key is test_row_0/B:col10/1731032590109/Put/seqid=0 2024-11-08T02:23:11,062 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411087ed72818f4554b0f8b735e6865d92c5e_0ecb51078105f519ce4baa47714771b2, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:11,062 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087ed72818f4554b0f8b735e6865d92c5e_0ecb51078105f519ce4baa47714771b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:11,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742002_1178 (size=12409) 2024-11-08T02:23:11,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742003_1179 (size=4469) 2024-11-08T02:23:11,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T02:23:11,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:11,170 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:23:11,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083e221f9b408c41018b784d7c93e454ee_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032590155/Put/seqid=0 2024-11-08T02:23:11,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742004_1180 (size=12304) 2024-11-08T02:23:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:11,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:11,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032651354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:11,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032651468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:11,471 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#A#compaction#153 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:11,472 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/c356c0d2d16f48c1a46f285a8657ca11 is 175, key is test_row_0/A:col10/1731032590109/Put/seqid=0 2024-11-08T02:23:11,475 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/47f20814eb85463789488526eec993df as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/47f20814eb85463789488526eec993df 2024-11-08T02:23:11,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742005_1181 (size=31363) 2024-11-08T02:23:11,481 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/B of 0ecb51078105f519ce4baa47714771b2 into 47f20814eb85463789488526eec993df(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:11,481 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:11,481 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/B, priority=13, startTime=1731032591049; duration=0sec 2024-11-08T02:23:11,481 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:11,481 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:B 2024-11-08T02:23:11,481 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:11,485 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:11,485 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/C is initiating minor compaction (all files) 2024-11-08T02:23:11,486 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/C in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:11,486 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/4d5495af73b545fb887ae37459244334, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ea77ad4937874145af048171939db707, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/07e246e21c0e4478bb5a326c032e7928] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=35.5 K 2024-11-08T02:23:11,486 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d5495af73b545fb887ae37459244334, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731032587008 2024-11-08T02:23:11,487 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting ea77ad4937874145af048171939db707, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731032587732 2024-11-08T02:23:11,487 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 07e246e21c0e4478bb5a326c032e7928, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731032588927 2024-11-08T02:23:11,489 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/c356c0d2d16f48c1a46f285a8657ca11 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c356c0d2d16f48c1a46f285a8657ca11 2024-11-08T02:23:11,495 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/A of 0ecb51078105f519ce4baa47714771b2 into c356c0d2d16f48c1a46f285a8657ca11(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:11,495 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:11,495 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/A, priority=13, startTime=1731032591049; duration=0sec 2024-11-08T02:23:11,496 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:11,496 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:A 2024-11-08T02:23:11,502 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#C#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:11,502 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/069f30ec9edd4f9185056fc8bf3ddc80 is 50, key is test_row_0/C:col10/1731032590109/Put/seqid=0 2024-11-08T02:23:11,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742006_1182 (size=12409) 2024-11-08T02:23:11,514 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/069f30ec9edd4f9185056fc8bf3ddc80 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/069f30ec9edd4f9185056fc8bf3ddc80 2024-11-08T02:23:11,520 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/C of 0ecb51078105f519ce4baa47714771b2 into 069f30ec9edd4f9185056fc8bf3ddc80(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:11,520 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:11,520 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/C, priority=13, startTime=1731032591049; duration=0sec 2024-11-08T02:23:11,520 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:11,520 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:C 2024-11-08T02:23:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T02:23:11,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:11,594 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083e221f9b408c41018b784d7c93e454ee_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083e221f9b408c41018b784d7c93e454ee_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:11,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/25404ecf405c4dfa803d9c27579e5405, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:11,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/25404ecf405c4dfa803d9c27579e5405 is 175, key is test_row_0/A:col10/1731032590155/Put/seqid=0 2024-11-08T02:23:11,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742007_1183 (size=31105) 2024-11-08T02:23:11,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032651678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:11,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:11,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:11,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032651988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:12,001 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/25404ecf405c4dfa803d9c27579e5405 2024-11-08T02:23:12,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/52142225e6cd4c0b95b13bfc47b2dab9 is 50, key is test_row_0/B:col10/1731032590155/Put/seqid=0 2024-11-08T02:23:12,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742008_1184 (size=12151) 2024-11-08T02:23:12,023 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/52142225e6cd4c0b95b13bfc47b2dab9 2024-11-08T02:23:12,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/430a06a1b82749f7983061769c57f85c is 50, key is test_row_0/C:col10/1731032590155/Put/seqid=0 2024-11-08T02:23:12,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742009_1185 (size=12151) 2024-11-08T02:23:12,036 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/430a06a1b82749f7983061769c57f85c 2024-11-08T02:23:12,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/25404ecf405c4dfa803d9c27579e5405 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/25404ecf405c4dfa803d9c27579e5405 2024-11-08T02:23:12,046 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/25404ecf405c4dfa803d9c27579e5405, entries=150, sequenceid=154, filesize=30.4 K 2024-11-08T02:23:12,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/52142225e6cd4c0b95b13bfc47b2dab9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/52142225e6cd4c0b95b13bfc47b2dab9 2024-11-08T02:23:12,054 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/52142225e6cd4c0b95b13bfc47b2dab9, entries=150, sequenceid=154, filesize=11.9 K 2024-11-08T02:23:12,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/430a06a1b82749f7983061769c57f85c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/430a06a1b82749f7983061769c57f85c 2024-11-08T02:23:12,061 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/430a06a1b82749f7983061769c57f85c, entries=150, sequenceid=154, filesize=11.9 K 2024-11-08T02:23:12,062 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 0ecb51078105f519ce4baa47714771b2 in 892ms, sequenceid=154, compaction requested=false 2024-11-08T02:23:12,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:12,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:12,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-08T02:23:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-08T02:23:12,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-08T02:23:12,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6620 sec 2024-11-08T02:23:12,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6660 sec 2024-11-08T02:23:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:12,501 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:12,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T02:23:12,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ea75596df68a4cfcb61223cdceb3b676_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032592500/Put/seqid=0 2024-11-08T02:23:12,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742010_1186 (size=14794) 2024-11-08T02:23:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T02:23:12,549 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:12,549 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-08T02:23:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T02:23:12,551 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:12,552 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:12,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:12,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032652580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:12,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:12,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:12,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T02:23:12,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032652688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:12,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:12,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T02:23:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:12,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:12,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:12,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T02:23:12,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:12,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:12,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:12,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:12,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T02:23:12,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032652898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:12,922 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:12,927 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ea75596df68a4cfcb61223cdceb3b676_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ea75596df68a4cfcb61223cdceb3b676_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:12,928 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/5963ff1f9ae1477e90aceb1a452b2c07, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:12,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/5963ff1f9ae1477e90aceb1a452b2c07 is 175, key is test_row_0/A:col10/1731032592500/Put/seqid=0 2024-11-08T02:23:12,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742011_1187 (size=39749) 2024-11-08T02:23:13,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T02:23:13,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:13,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T02:23:13,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:13,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T02:23:13,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:13,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032653207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:13,260 INFO [master/331c7316141f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-08T02:23:13,260 INFO [master/331c7316141f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-08T02:23:13,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T02:23:13,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:13,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,332 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/5963ff1f9ae1477e90aceb1a452b2c07 2024-11-08T02:23:13,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/6a5186dfc9a14e979a032ca2fb93d780 is 50, key is test_row_0/B:col10/1731032592500/Put/seqid=0 2024-11-08T02:23:13,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742012_1188 (size=12151) 2024-11-08T02:23:13,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032653418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:13,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032653418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:13,421 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:13,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:13,421 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:13,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:13,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032653427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:13,428 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:13,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032653458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:13,461 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:13,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T02:23:13,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:13,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T02:23:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T02:23:13,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032653718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:13,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/6a5186dfc9a14e979a032ca2fb93d780 2024-11-08T02:23:13,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/2bdd73c4c8c6496eab79b5353bed3319 is 50, key is test_row_0/C:col10/1731032592500/Put/seqid=0 2024-11-08T02:23:13,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742013_1189 (size=12151) 2024-11-08T02:23:13,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T02:23:13,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:13,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T02:23:13,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:13,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:13,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:14,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T02:23:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:14,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:14,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:14,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/2bdd73c4c8c6496eab79b5353bed3319 2024-11-08T02:23:14,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/5963ff1f9ae1477e90aceb1a452b2c07 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5963ff1f9ae1477e90aceb1a452b2c07 2024-11-08T02:23:14,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5963ff1f9ae1477e90aceb1a452b2c07, entries=200, sequenceid=174, filesize=38.8 K 2024-11-08T02:23:14,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/6a5186dfc9a14e979a032ca2fb93d780 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6a5186dfc9a14e979a032ca2fb93d780 2024-11-08T02:23:14,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6a5186dfc9a14e979a032ca2fb93d780, entries=150, sequenceid=174, filesize=11.9 K 2024-11-08T02:23:14,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/2bdd73c4c8c6496eab79b5353bed3319 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/2bdd73c4c8c6496eab79b5353bed3319 2024-11-08T02:23:14,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/2bdd73c4c8c6496eab79b5353bed3319, entries=150, sequenceid=174, filesize=11.9 K 2024-11-08T02:23:14,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 0ecb51078105f519ce4baa47714771b2 in 1680ms, sequenceid=174, compaction requested=true 2024-11-08T02:23:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:14,181 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:14,181 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:14,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:14,182 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:14,182 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:14,182 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/A is initiating minor compaction (all files) 2024-11-08T02:23:14,182 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/B is initiating minor compaction (all files) 2024-11-08T02:23:14,182 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/A in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:14,182 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/B in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:14,182 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c356c0d2d16f48c1a46f285a8657ca11, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/25404ecf405c4dfa803d9c27579e5405, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5963ff1f9ae1477e90aceb1a452b2c07] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=99.8 K 2024-11-08T02:23:14,182 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/47f20814eb85463789488526eec993df, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/52142225e6cd4c0b95b13bfc47b2dab9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6a5186dfc9a14e979a032ca2fb93d780] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=35.9 K 2024-11-08T02:23:14,182 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:14,182 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c356c0d2d16f48c1a46f285a8657ca11, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/25404ecf405c4dfa803d9c27579e5405, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5963ff1f9ae1477e90aceb1a452b2c07] 2024-11-08T02:23:14,183 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 47f20814eb85463789488526eec993df, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731032588927 2024-11-08T02:23:14,183 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c356c0d2d16f48c1a46f285a8657ca11, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731032588927 2024-11-08T02:23:14,183 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 52142225e6cd4c0b95b13bfc47b2dab9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731032590155 2024-11-08T02:23:14,183 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25404ecf405c4dfa803d9c27579e5405, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731032590155 2024-11-08T02:23:14,183 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a5186dfc9a14e979a032ca2fb93d780, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731032591338 2024-11-08T02:23:14,183 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5963ff1f9ae1477e90aceb1a452b2c07, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731032591338 2024-11-08T02:23:14,189 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:14,190 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#B#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:14,191 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/106899eed2d646baaa7c82d2ab1ffca0 is 50, key is test_row_0/B:col10/1731032592500/Put/seqid=0 2024-11-08T02:23:14,192 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110882eb5746a3fe4ca998b1591a91401f07_0ecb51078105f519ce4baa47714771b2 store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:14,193 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110882eb5746a3fe4ca998b1591a91401f07_0ecb51078105f519ce4baa47714771b2, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:14,193 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110882eb5746a3fe4ca998b1591a91401f07_0ecb51078105f519ce4baa47714771b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:14,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742014_1190 (size=12561) 2024-11-08T02:23:14,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742015_1191 (size=4469) 2024-11-08T02:23:14,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T02:23:14,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:14,232 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T02:23:14,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108325a6eb6ebf84e33ab64273ac0294aa0_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032592574/Put/seqid=0 2024-11-08T02:23:14,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742016_1192 (size=12304) 2024-11-08T02:23:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:14,245 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108325a6eb6ebf84e33ab64273ac0294aa0_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108325a6eb6ebf84e33ab64273ac0294aa0_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:14,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/29c28ef0505b4616833e39314fff27a9, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:14,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/29c28ef0505b4616833e39314fff27a9 is 175, key is test_row_0/A:col10/1731032592574/Put/seqid=0 2024-11-08T02:23:14,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742017_1193 (size=31105) 2024-11-08T02:23:14,602 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/106899eed2d646baaa7c82d2ab1ffca0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/106899eed2d646baaa7c82d2ab1ffca0 2024-11-08T02:23:14,607 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/B of 0ecb51078105f519ce4baa47714771b2 into 106899eed2d646baaa7c82d2ab1ffca0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:14,607 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:14,607 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/B, priority=13, startTime=1731032594181; duration=0sec 2024-11-08T02:23:14,607 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:14,607 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:B 2024-11-08T02:23:14,608 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:14,609 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:14,609 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/C is initiating minor compaction (all files) 2024-11-08T02:23:14,609 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/C in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:14,609 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/069f30ec9edd4f9185056fc8bf3ddc80, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/430a06a1b82749f7983061769c57f85c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/2bdd73c4c8c6496eab79b5353bed3319] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=35.9 K 2024-11-08T02:23:14,609 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 069f30ec9edd4f9185056fc8bf3ddc80, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731032588927 2024-11-08T02:23:14,610 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 430a06a1b82749f7983061769c57f85c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731032590155 2024-11-08T02:23:14,610 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bdd73c4c8c6496eab79b5353bed3319, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731032591338 2024-11-08T02:23:14,611 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#A#compaction#163 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:14,611 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/ce1ab94f2a5a4f1f98c03bd5d88a5865 is 175, key is test_row_0/A:col10/1731032592500/Put/seqid=0 2024-11-08T02:23:14,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742018_1194 (size=31515) 2024-11-08T02:23:14,621 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:14,621 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/cd87c82ac6104b189f5277d5e2519f0f is 50, key is test_row_0/C:col10/1731032592500/Put/seqid=0 2024-11-08T02:23:14,623 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/ce1ab94f2a5a4f1f98c03bd5d88a5865 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/ce1ab94f2a5a4f1f98c03bd5d88a5865 2024-11-08T02:23:14,629 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/A of 0ecb51078105f519ce4baa47714771b2 into ce1ab94f2a5a4f1f98c03bd5d88a5865(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:14,629 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:14,629 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/A, priority=13, startTime=1731032594181; duration=0sec 2024-11-08T02:23:14,629 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:14,629 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:A 2024-11-08T02:23:14,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742019_1195 (size=12561) 2024-11-08T02:23:14,655 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/29c28ef0505b4616833e39314fff27a9 2024-11-08T02:23:14,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/62c5de302df64e17870f7e637faef5ca is 50, key is test_row_0/B:col10/1731032592574/Put/seqid=0 2024-11-08T02:23:14,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742020_1196 (size=12151) 2024-11-08T02:23:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T02:23:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:14,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:14,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032654782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:14,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032654887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:14,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:14,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:14,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:15,037 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/cd87c82ac6104b189f5277d5e2519f0f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/cd87c82ac6104b189f5277d5e2519f0f 2024-11-08T02:23:15,043 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/C of 0ecb51078105f519ce4baa47714771b2 into cd87c82ac6104b189f5277d5e2519f0f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:15,043 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:15,043 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/C, priority=13, startTime=1731032594181; duration=0sec 2024-11-08T02:23:15,043 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:15,043 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:C 2024-11-08T02:23:15,068 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/62c5de302df64e17870f7e637faef5ca 2024-11-08T02:23:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/60a100524734461e82ba59bf73ac428b is 50, key is test_row_0/C:col10/1731032592574/Put/seqid=0 2024-11-08T02:23:15,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742021_1197 (size=12151) 2024-11-08T02:23:15,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032655098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:15,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:15,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032655408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:15,481 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/60a100524734461e82ba59bf73ac428b 2024-11-08T02:23:15,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/29c28ef0505b4616833e39314fff27a9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/29c28ef0505b4616833e39314fff27a9 2024-11-08T02:23:15,490 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/29c28ef0505b4616833e39314fff27a9, entries=150, sequenceid=193, filesize=30.4 K 2024-11-08T02:23:15,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/62c5de302df64e17870f7e637faef5ca as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/62c5de302df64e17870f7e637faef5ca 2024-11-08T02:23:15,496 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/62c5de302df64e17870f7e637faef5ca, entries=150, sequenceid=193, filesize=11.9 K 2024-11-08T02:23:15,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/60a100524734461e82ba59bf73ac428b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/60a100524734461e82ba59bf73ac428b 2024-11-08T02:23:15,502 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/60a100524734461e82ba59bf73ac428b, entries=150, sequenceid=193, filesize=11.9 K 2024-11-08T02:23:15,502 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 0ecb51078105f519ce4baa47714771b2 in 1270ms, sequenceid=193, compaction requested=false 2024-11-08T02:23:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-08T02:23:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-08T02:23:15,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-08T02:23:15,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9520 sec 2024-11-08T02:23:15,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.9570 sec 2024-11-08T02:23:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:15,919 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:15,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T02:23:15,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081a957cca1c2e41b690885f0aa72131aa_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032594757/Put/seqid=0 2024-11-08T02:23:15,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742022_1198 (size=14794) 2024-11-08T02:23:15,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032655963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:15,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:15,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:15,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:16,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032656068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:16,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:16,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032656277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:16,331 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:16,378 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081a957cca1c2e41b690885f0aa72131aa_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081a957cca1c2e41b690885f0aa72131aa_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:16,379 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/d0e7bb55fd8542698c3a692969045e30, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:16,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/d0e7bb55fd8542698c3a692969045e30 is 175, key is test_row_0/A:col10/1731032594757/Put/seqid=0 2024-11-08T02:23:16,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742023_1199 (size=39749) 2024-11-08T02:23:16,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032656587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T02:23:16,708 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:16,708 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-08T02:23:16,710 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T02:23:16,710 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:16,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:16,783 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/d0e7bb55fd8542698c3a692969045e30 2024-11-08T02:23:16,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/143bd6d81944432c8220fec1706759b5 is 50, key is test_row_0/B:col10/1731032594757/Put/seqid=0 2024-11-08T02:23:16,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742024_1200 (size=12151) 2024-11-08T02:23:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T02:23:16,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T02:23:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:16,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T02:23:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T02:23:17,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032657098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:17,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T02:23:17,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:17,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/143bd6d81944432c8220fec1706759b5 2024-11-08T02:23:17,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/fc05a157dc174008bfd8e9a8e03d11ed is 50, key is test_row_0/C:col10/1731032594757/Put/seqid=0 2024-11-08T02:23:17,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742025_1201 (size=12151) 2024-11-08T02:23:17,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T02:23:17,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:17,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T02:23:17,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T02:23:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/fc05a157dc174008bfd8e9a8e03d11ed 2024-11-08T02:23:17,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/d0e7bb55fd8542698c3a692969045e30 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/d0e7bb55fd8542698c3a692969045e30 2024-11-08T02:23:17,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/d0e7bb55fd8542698c3a692969045e30, entries=200, sequenceid=214, filesize=38.8 K 2024-11-08T02:23:17,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/143bd6d81944432c8220fec1706759b5 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/143bd6d81944432c8220fec1706759b5 2024-11-08T02:23:17,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/143bd6d81944432c8220fec1706759b5, entries=150, sequenceid=214, filesize=11.9 K 2024-11-08T02:23:17,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/fc05a157dc174008bfd8e9a8e03d11ed as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/fc05a157dc174008bfd8e9a8e03d11ed 2024-11-08T02:23:17,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T02:23:17,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:17,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:17,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/fc05a157dc174008bfd8e9a8e03d11ed, entries=150, sequenceid=214, filesize=11.9 K 2024-11-08T02:23:17,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 0ecb51078105f519ce4baa47714771b2 in 1714ms, sequenceid=214, compaction requested=true 2024-11-08T02:23:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:17,634 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:17,634 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:17,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:17,635 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:17,635 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:17,635 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/B is initiating minor compaction (all files) 2024-11-08T02:23:17,635 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/A is initiating minor compaction (all files) 2024-11-08T02:23:17,635 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/B in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,635 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/A in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,635 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/ce1ab94f2a5a4f1f98c03bd5d88a5865, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/29c28ef0505b4616833e39314fff27a9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/d0e7bb55fd8542698c3a692969045e30] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=100.0 K 2024-11-08T02:23:17,635 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/106899eed2d646baaa7c82d2ab1ffca0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/62c5de302df64e17870f7e637faef5ca, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/143bd6d81944432c8220fec1706759b5] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=36.0 K 2024-11-08T02:23:17,635 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,635 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/ce1ab94f2a5a4f1f98c03bd5d88a5865, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/29c28ef0505b4616833e39314fff27a9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/d0e7bb55fd8542698c3a692969045e30] 2024-11-08T02:23:17,636 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 106899eed2d646baaa7c82d2ab1ffca0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731032591338 2024-11-08T02:23:17,636 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce1ab94f2a5a4f1f98c03bd5d88a5865, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731032591338 2024-11-08T02:23:17,636 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 62c5de302df64e17870f7e637faef5ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731032592570 2024-11-08T02:23:17,636 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29c28ef0505b4616833e39314fff27a9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731032592570 2024-11-08T02:23:17,637 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 143bd6d81944432c8220fec1706759b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731032594757 2024-11-08T02:23:17,637 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0e7bb55fd8542698c3a692969045e30, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731032594757 2024-11-08T02:23:17,645 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:17,645 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/9301fead6acb4f7587e318e2898e0150 is 50, key is test_row_0/B:col10/1731032594757/Put/seqid=0 2024-11-08T02:23:17,647 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:17,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742026_1202 (size=12663) 2024-11-08T02:23:17,658 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/9301fead6acb4f7587e318e2898e0150 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/9301fead6acb4f7587e318e2898e0150 2024-11-08T02:23:17,663 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/B of 0ecb51078105f519ce4baa47714771b2 into 9301fead6acb4f7587e318e2898e0150(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:17,663 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:17,663 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/B, priority=13, startTime=1731032597634; duration=0sec 2024-11-08T02:23:17,663 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:17,663 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:B 2024-11-08T02:23:17,663 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:17,665 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:17,665 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/C is initiating minor compaction (all files) 2024-11-08T02:23:17,665 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/C in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,665 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/cd87c82ac6104b189f5277d5e2519f0f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/60a100524734461e82ba59bf73ac428b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/fc05a157dc174008bfd8e9a8e03d11ed] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=36.0 K 2024-11-08T02:23:17,666 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting cd87c82ac6104b189f5277d5e2519f0f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731032591338 2024-11-08T02:23:17,666 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 60a100524734461e82ba59bf73ac428b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731032592570 2024-11-08T02:23:17,666 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting fc05a157dc174008bfd8e9a8e03d11ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731032594757 2024-11-08T02:23:17,673 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108c7d8f28d299041ff8ed1d9e8952b0c00_0ecb51078105f519ce4baa47714771b2 store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:17,675 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108c7d8f28d299041ff8ed1d9e8952b0c00_0ecb51078105f519ce4baa47714771b2, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:17,675 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c7d8f28d299041ff8ed1d9e8952b0c00_0ecb51078105f519ce4baa47714771b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:17,682 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#C#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:17,683 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/6ca4e11074fb4d6eaad3aee8d99ccaaa is 50, key is test_row_0/C:col10/1731032594757/Put/seqid=0 2024-11-08T02:23:17,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742028_1204 (size=12663) 2024-11-08T02:23:17,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742027_1203 (size=4469) 2024-11-08T02:23:17,694 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/6ca4e11074fb4d6eaad3aee8d99ccaaa as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/6ca4e11074fb4d6eaad3aee8d99ccaaa 2024-11-08T02:23:17,695 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#A#compaction#172 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:17,696 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/afdf9aae7ef14450bba9a929ab35bd42 is 175, key is test_row_0/A:col10/1731032594757/Put/seqid=0 2024-11-08T02:23:17,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742029_1205 (size=31617) 2024-11-08T02:23:17,702 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/C of 0ecb51078105f519ce4baa47714771b2 into 6ca4e11074fb4d6eaad3aee8d99ccaaa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:17,702 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:17,702 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/C, priority=13, startTime=1731032597634; duration=0sec 2024-11-08T02:23:17,702 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:17,703 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:C 2024-11-08T02:23:17,711 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/afdf9aae7ef14450bba9a929ab35bd42 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/afdf9aae7ef14450bba9a929ab35bd42 2024-11-08T02:23:17,718 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/A of 0ecb51078105f519ce4baa47714771b2 into afdf9aae7ef14450bba9a929ab35bd42(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:17,718 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:17,718 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/A, priority=13, startTime=1731032597634; duration=0sec 2024-11-08T02:23:17,718 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:17,718 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:A 2024-11-08T02:23:17,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T02:23:17,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:17,781 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T02:23:17,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083be390d128e244c6a93cb6b6b71204b5_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032595962/Put/seqid=0 2024-11-08T02:23:17,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742030_1206 (size=12304) 2024-11-08T02:23:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T02:23:18,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:18,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:18,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:18,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032658154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:18,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:18,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:18,196 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083be390d128e244c6a93cb6b6b71204b5_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083be390d128e244c6a93cb6b6b71204b5_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:18,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/72d00e7c19cf465b9dc178d2cf031290, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:18,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/72d00e7c19cf465b9dc178d2cf031290 is 175, key is test_row_0/A:col10/1731032595962/Put/seqid=0 2024-11-08T02:23:18,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742031_1207 (size=31105) 2024-11-08T02:23:18,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032658268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:18,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:18,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:18,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:18,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032658477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:18,603 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/72d00e7c19cf465b9dc178d2cf031290 2024-11-08T02:23:18,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/ad90b6ea198a4b5d80f7111e15d87355 is 50, key is test_row_0/B:col10/1731032595962/Put/seqid=0 2024-11-08T02:23:18,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742032_1208 (size=12151) 2024-11-08T02:23:18,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032658788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T02:23:19,018 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/ad90b6ea198a4b5d80f7111e15d87355 2024-11-08T02:23:19,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/d14246d9ac654e64a899df68511b09ae is 50, key is test_row_0/C:col10/1731032595962/Put/seqid=0 2024-11-08T02:23:19,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742033_1209 (size=12151) 2024-11-08T02:23:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032659298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:19,430 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/d14246d9ac654e64a899df68511b09ae 2024-11-08T02:23:19,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/72d00e7c19cf465b9dc178d2cf031290 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/72d00e7c19cf465b9dc178d2cf031290 2024-11-08T02:23:19,438 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/72d00e7c19cf465b9dc178d2cf031290, entries=150, sequenceid=235, filesize=30.4 K 2024-11-08T02:23:19,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/ad90b6ea198a4b5d80f7111e15d87355 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ad90b6ea198a4b5d80f7111e15d87355 2024-11-08T02:23:19,443 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ad90b6ea198a4b5d80f7111e15d87355, entries=150, sequenceid=235, filesize=11.9 K 2024-11-08T02:23:19,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/d14246d9ac654e64a899df68511b09ae as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d14246d9ac654e64a899df68511b09ae 2024-11-08T02:23:19,448 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d14246d9ac654e64a899df68511b09ae, entries=150, sequenceid=235, filesize=11.9 K 2024-11-08T02:23:19,449 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 0ecb51078105f519ce4baa47714771b2 in 1668ms, sequenceid=235, compaction requested=false 2024-11-08T02:23:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-08T02:23:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-08T02:23:19,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-08T02:23:19,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7400 sec 2024-11-08T02:23:19,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.7450 sec 2024-11-08T02:23:20,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:20,311 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:20,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T02:23:20,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cfe95ab77c57496d916bd63bb399a3f4_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032600310/Put/seqid=0 2024-11-08T02:23:20,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742034_1210 (size=14794) 2024-11-08T02:23:20,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032660350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:20,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032660458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:20,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032660668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:20,723 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:20,727 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cfe95ab77c57496d916bd63bb399a3f4_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cfe95ab77c57496d916bd63bb399a3f4_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:20,728 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/456c153b68e44942b8f18dc5542c936c, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:20,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/456c153b68e44942b8f18dc5542c936c is 175, key is test_row_0/A:col10/1731032600310/Put/seqid=0 2024-11-08T02:23:20,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742035_1211 (size=39749) 2024-11-08T02:23:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T02:23:20,879 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:20,879 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-08T02:23:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T02:23:20,881 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:20,881 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:20,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:20,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032660978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T02:23:21,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T02:23:21,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,133 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/456c153b68e44942b8f18dc5542c936c 2024-11-08T02:23:21,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/ceaea4875c564d789eaa9d24bca58300 is 50, key is test_row_0/B:col10/1731032600310/Put/seqid=0 2024-11-08T02:23:21,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742036_1212 (size=12151) 2024-11-08T02:23:21,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T02:23:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T02:23:21,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T02:23:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032661488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:21,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T02:23:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T02:23:21,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/ceaea4875c564d789eaa9d24bca58300 2024-11-08T02:23:21,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/09d687f0c8644ce9869f9fa4add73a04 is 50, key is test_row_0/C:col10/1731032600310/Put/seqid=0 2024-11-08T02:23:21,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742037_1213 (size=12151) 2024-11-08T02:23:21,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T02:23:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T02:23:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T02:23:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:21,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:21,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/09d687f0c8644ce9869f9fa4add73a04 2024-11-08T02:23:21,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/456c153b68e44942b8f18dc5542c936c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/456c153b68e44942b8f18dc5542c936c 2024-11-08T02:23:21,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/456c153b68e44942b8f18dc5542c936c, entries=200, sequenceid=254, filesize=38.8 K 2024-11-08T02:23:21,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/ceaea4875c564d789eaa9d24bca58300 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ceaea4875c564d789eaa9d24bca58300 2024-11-08T02:23:21,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ceaea4875c564d789eaa9d24bca58300, entries=150, sequenceid=254, filesize=11.9 K 2024-11-08T02:23:21,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/09d687f0c8644ce9869f9fa4add73a04 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/09d687f0c8644ce9869f9fa4add73a04 2024-11-08T02:23:22,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/09d687f0c8644ce9869f9fa4add73a04, entries=150, sequenceid=254, filesize=11.9 K 2024-11-08T02:23:22,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 0ecb51078105f519ce4baa47714771b2 in 1691ms, sequenceid=254, compaction requested=true 2024-11-08T02:23:22,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:22,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:22,002 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:22,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:22,002 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:22,004 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:22,004 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:22,004 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/A is initiating minor compaction (all files) 2024-11-08T02:23:22,004 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/B is initiating minor compaction (all files) 2024-11-08T02:23:22,004 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/A in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:22,005 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/B in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:22,005 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/9301fead6acb4f7587e318e2898e0150, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ad90b6ea198a4b5d80f7111e15d87355, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ceaea4875c564d789eaa9d24bca58300] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=36.1 K 2024-11-08T02:23:22,005 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/afdf9aae7ef14450bba9a929ab35bd42, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/72d00e7c19cf465b9dc178d2cf031290, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/456c153b68e44942b8f18dc5542c936c] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=100.1 K 2024-11-08T02:23:22,005 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:22,005 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/afdf9aae7ef14450bba9a929ab35bd42, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/72d00e7c19cf465b9dc178d2cf031290, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/456c153b68e44942b8f18dc5542c936c] 2024-11-08T02:23:22,005 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 9301fead6acb4f7587e318e2898e0150, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731032594757 2024-11-08T02:23:22,005 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting afdf9aae7ef14450bba9a929ab35bd42, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731032594757 2024-11-08T02:23:22,005 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting ad90b6ea198a4b5d80f7111e15d87355, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032595944 2024-11-08T02:23:22,006 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72d00e7c19cf465b9dc178d2cf031290, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032595944 2024-11-08T02:23:22,006 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting ceaea4875c564d789eaa9d24bca58300, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731032598137 2024-11-08T02:23:22,006 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 456c153b68e44942b8f18dc5542c936c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731032598137 2024-11-08T02:23:22,014 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:22,017 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#B#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:22,017 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/23d278d8c6f7495e926e4d9099517535 is 50, key is test_row_0/B:col10/1731032600310/Put/seqid=0 2024-11-08T02:23:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T02:23:22,020 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110858bec8b44cf44898b66b1e7775e595d7_0ecb51078105f519ce4baa47714771b2 store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:22,022 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110858bec8b44cf44898b66b1e7775e595d7_0ecb51078105f519ce4baa47714771b2, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:22,022 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110858bec8b44cf44898b66b1e7775e595d7_0ecb51078105f519ce4baa47714771b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:22,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742038_1214 (size=12765) 2024-11-08T02:23:22,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742039_1215 (size=4469) 2024-11-08T02:23:22,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T02:23:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:22,103 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T02:23:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085fd47e845d8a4190a484d15302d912fd_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032600349/Put/seqid=0 2024-11-08T02:23:22,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742040_1216 (size=12454) 2024-11-08T02:23:22,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:22,144 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085fd47e845d8a4190a484d15302d912fd_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085fd47e845d8a4190a484d15302d912fd_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/f1334b58fde14e738685967e34acb69c, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/f1334b58fde14e738685967e34acb69c is 175, key is test_row_0/A:col10/1731032600349/Put/seqid=0 2024-11-08T02:23:22,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742041_1217 (size=31255) 2024-11-08T02:23:22,431 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/23d278d8c6f7495e926e4d9099517535 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/23d278d8c6f7495e926e4d9099517535 2024-11-08T02:23:22,434 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#A#compaction#180 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:22,435 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/f4621923375948b8b8730b85d89e90b8 is 175, key is test_row_0/A:col10/1731032600310/Put/seqid=0 2024-11-08T02:23:22,439 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/B of 0ecb51078105f519ce4baa47714771b2 into 23d278d8c6f7495e926e4d9099517535(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:22,439 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:22,439 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/B, priority=13, startTime=1731032602002; duration=0sec 2024-11-08T02:23:22,439 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:22,439 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:B 2024-11-08T02:23:22,439 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:22,441 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:22,441 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/C is initiating minor compaction (all files) 2024-11-08T02:23:22,441 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/C in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:22,441 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/6ca4e11074fb4d6eaad3aee8d99ccaaa, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d14246d9ac654e64a899df68511b09ae, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/09d687f0c8644ce9869f9fa4add73a04] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=36.1 K 2024-11-08T02:23:22,442 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ca4e11074fb4d6eaad3aee8d99ccaaa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731032594757 2024-11-08T02:23:22,442 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting d14246d9ac654e64a899df68511b09ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032595944 2024-11-08T02:23:22,443 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d687f0c8644ce9869f9fa4add73a04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731032598137 2024-11-08T02:23:22,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742042_1218 (size=31719) 2024-11-08T02:23:22,454 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#C#compaction#183 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:22,454 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/15e120ae87054579934adbd666208370 is 50, key is test_row_0/C:col10/1731032600310/Put/seqid=0 2024-11-08T02:23:22,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742043_1219 (size=12765) 2024-11-08T02:23:22,466 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/15e120ae87054579934adbd666208370 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/15e120ae87054579934adbd666208370 2024-11-08T02:23:22,472 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/C of 0ecb51078105f519ce4baa47714771b2 into 15e120ae87054579934adbd666208370(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:22,472 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:22,472 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/C, priority=13, startTime=1731032602003; duration=0sec 2024-11-08T02:23:22,472 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:22,472 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:C 2024-11-08T02:23:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. as already flushing 2024-11-08T02:23:22,555 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/f1334b58fde14e738685967e34acb69c 2024-11-08T02:23:22,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/1832b53f6ac345f5a3ad29ebea0a2858 is 50, key is test_row_0/B:col10/1731032600349/Put/seqid=0 2024-11-08T02:23:22,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032662570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:22,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742044_1220 (size=12301) 2024-11-08T02:23:22,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032662678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:22,857 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/f4621923375948b8b8730b85d89e90b8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f4621923375948b8b8730b85d89e90b8 2024-11-08T02:23:22,863 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/A of 0ecb51078105f519ce4baa47714771b2 into f4621923375948b8b8730b85d89e90b8(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:22,863 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:22,863 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/A, priority=13, startTime=1731032602002; duration=0sec 2024-11-08T02:23:22,864 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:22,864 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:A 2024-11-08T02:23:22,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032662888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:22,978 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/1832b53f6ac345f5a3ad29ebea0a2858 2024-11-08T02:23:22,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/9bd0d97cebf54c5ab4bd4fdb5ee0aab4 is 50, key is test_row_0/C:col10/1731032600349/Put/seqid=0 2024-11-08T02:23:23,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742045_1221 (size=12301) 2024-11-08T02:23:23,010 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/9bd0d97cebf54c5ab4bd4fdb5ee0aab4 2024-11-08T02:23:23,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/f1334b58fde14e738685967e34acb69c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f1334b58fde14e738685967e34acb69c 2024-11-08T02:23:23,021 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f1334b58fde14e738685967e34acb69c, entries=150, sequenceid=271, filesize=30.5 K 2024-11-08T02:23:23,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/1832b53f6ac345f5a3ad29ebea0a2858 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1832b53f6ac345f5a3ad29ebea0a2858 2024-11-08T02:23:23,028 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1832b53f6ac345f5a3ad29ebea0a2858, entries=150, sequenceid=271, filesize=12.0 K 2024-11-08T02:23:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T02:23:23,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/9bd0d97cebf54c5ab4bd4fdb5ee0aab4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/9bd0d97cebf54c5ab4bd4fdb5ee0aab4 2024-11-08T02:23:23,034 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/9bd0d97cebf54c5ab4bd4fdb5ee0aab4, entries=150, sequenceid=271, filesize=12.0 K 2024-11-08T02:23:23,035 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 0ecb51078105f519ce4baa47714771b2 in 932ms, sequenceid=271, compaction requested=false 2024-11-08T02:23:23,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:23,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:23,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-08T02:23:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-08T02:23:23,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-08T02:23:23,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1570 sec 2024-11-08T02:23:23,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.1620 sec 2024-11-08T02:23:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:23,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:23:23,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ecd38f4a5ea44f2983fefb02d71ede3f_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032602543/Put/seqid=0 2024-11-08T02:23:23,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742046_1222 (size=12454) 2024-11-08T02:23:23,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032663258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:23,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:23,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032663368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44892 deadline: 1731032663468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:23,470 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:23,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44912 deadline: 1731032663499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:23,501 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:23,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44922 deadline: 1731032663508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:23,511 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:23,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44880 deadline: 1731032663548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:23,552 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 0ecb51078105f519ce4baa47714771b2 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18319 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:23,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:23,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032663578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:23,644 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:23,649 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ecd38f4a5ea44f2983fefb02d71ede3f_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ecd38f4a5ea44f2983fefb02d71ede3f_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:23,651 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/35c758025ae64274992bf4b7f5c2b593, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:23,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/35c758025ae64274992bf4b7f5c2b593 is 175, key is test_row_0/A:col10/1731032602543/Put/seqid=0 2024-11-08T02:23:23,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742047_1223 (size=31255) 2024-11-08T02:23:23,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032663888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:23,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:23,946 INFO [Thread-706 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-706. 2024-11-08T02:23:23,946 DEBUG [Thread-706 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:23,946 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:23,946 DEBUG [Thread-706 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:23,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:23,947 INFO [Thread-708 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-708. 2024-11-08T02:23:23,947 DEBUG [Thread-708 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:23,947 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:23,947 DEBUG [Thread-708 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:23,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:23,953 INFO [Thread-710 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-710. 2024-11-08T02:23:23,953 DEBUG [Thread-710 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:23,953 DEBUG [Thread-710 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:23,954 DEBUG [Thread-710 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:23,954 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:23,957 INFO [Thread-712 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-712. 2024-11-08T02:23:23,957 DEBUG [Thread-712 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:23,957 DEBUG [Thread-712 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:23,957 DEBUG [Thread-712 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:23,957 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:24,061 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/35c758025ae64274992bf4b7f5c2b593 2024-11-08T02:23:24,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/e9f75bac95cc4da387359a679f816eee is 50, key is test_row_0/B:col10/1731032602543/Put/seqid=0 2024-11-08T02:23:24,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742048_1224 (size=12301) 2024-11-08T02:23:24,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44936 deadline: 1731032664408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:23:24,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/e9f75bac95cc4da387359a679f816eee 2024-11-08T02:23:24,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/125363c13d9b4cbf9c867d25776f7167 is 50, key is test_row_0/C:col10/1731032602543/Put/seqid=0 2024-11-08T02:23:24,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742049_1225 (size=12301) 2024-11-08T02:23:24,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/125363c13d9b4cbf9c867d25776f7167 2024-11-08T02:23:24,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/35c758025ae64274992bf4b7f5c2b593 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/35c758025ae64274992bf4b7f5c2b593 2024-11-08T02:23:24,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/35c758025ae64274992bf4b7f5c2b593, entries=150, sequenceid=294, filesize=30.5 K 2024-11-08T02:23:24,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/e9f75bac95cc4da387359a679f816eee as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e9f75bac95cc4da387359a679f816eee 2024-11-08T02:23:24,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e9f75bac95cc4da387359a679f816eee, entries=150, sequenceid=294, filesize=12.0 K 2024-11-08T02:23:24,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/125363c13d9b4cbf9c867d25776f7167 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/125363c13d9b4cbf9c867d25776f7167 2024-11-08T02:23:24,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/125363c13d9b4cbf9c867d25776f7167, entries=150, sequenceid=294, filesize=12.0 K 2024-11-08T02:23:24,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 0ecb51078105f519ce4baa47714771b2 in 1710ms, sequenceid=294, compaction requested=true 2024-11-08T02:23:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:24,909 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:24,909 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:24,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ecb51078105f519ce4baa47714771b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:24,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:24,911 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:24,911 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:24,911 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/B is initiating minor compaction (all files) 2024-11-08T02:23:24,911 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/A is initiating minor compaction (all files) 2024-11-08T02:23:24,911 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/B in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:24,911 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/A in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:24,911 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/23d278d8c6f7495e926e4d9099517535, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1832b53f6ac345f5a3ad29ebea0a2858, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e9f75bac95cc4da387359a679f816eee] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=36.5 K 2024-11-08T02:23:24,911 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f4621923375948b8b8730b85d89e90b8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f1334b58fde14e738685967e34acb69c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/35c758025ae64274992bf4b7f5c2b593] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=92.0 K 2024-11-08T02:23:24,911 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:24,911 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f4621923375948b8b8730b85d89e90b8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f1334b58fde14e738685967e34acb69c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/35c758025ae64274992bf4b7f5c2b593] 2024-11-08T02:23:24,911 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 23d278d8c6f7495e926e4d9099517535, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731032598137 2024-11-08T02:23:24,912 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4621923375948b8b8730b85d89e90b8, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731032598137 2024-11-08T02:23:24,912 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 1832b53f6ac345f5a3ad29ebea0a2858, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731032600330 2024-11-08T02:23:24,912 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1334b58fde14e738685967e34acb69c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731032600330 2024-11-08T02:23:24,912 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting e9f75bac95cc4da387359a679f816eee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731032602543 2024-11-08T02:23:24,912 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35c758025ae64274992bf4b7f5c2b593, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731032602543 2024-11-08T02:23:24,924 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:24,925 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#B#compaction#189 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:24,926 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/1c25fd9df49e4322ae7e0058cd155513 is 50, key is test_row_0/B:col10/1731032602543/Put/seqid=0 2024-11-08T02:23:24,927 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108c6905a47abdb4b12a663c47e5dbfd825_0ecb51078105f519ce4baa47714771b2 store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:24,961 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108c6905a47abdb4b12a663c47e5dbfd825_0ecb51078105f519ce4baa47714771b2, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:24,961 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c6905a47abdb4b12a663c47e5dbfd825_0ecb51078105f519ce4baa47714771b2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:24,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742050_1226 (size=13017) 2024-11-08T02:23:25,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742051_1227 (size=4469) 2024-11-08T02:23:25,013 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#A#compaction#190 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:25,014 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/9d098a6b9e3948408f1ea7e518c1f060 is 175, key is test_row_0/A:col10/1731032602543/Put/seqid=0 2024-11-08T02:23:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T02:23:25,049 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:25,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742052_1228 (size=31971) 2024-11-08T02:23:25,380 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/1c25fd9df49e4322ae7e0058cd155513 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1c25fd9df49e4322ae7e0058cd155513 2024-11-08T02:23:25,385 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/B of 0ecb51078105f519ce4baa47714771b2 into 1c25fd9df49e4322ae7e0058cd155513(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:25,385 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:25,385 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/B, priority=13, startTime=1731032604909; duration=0sec 2024-11-08T02:23:25,385 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:25,385 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:B 2024-11-08T02:23:25,385 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:25,386 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:25,386 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 0ecb51078105f519ce4baa47714771b2/C is initiating minor compaction (all files) 2024-11-08T02:23:25,386 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ecb51078105f519ce4baa47714771b2/C in TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:25,387 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/15e120ae87054579934adbd666208370, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/9bd0d97cebf54c5ab4bd4fdb5ee0aab4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/125363c13d9b4cbf9c867d25776f7167] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp, totalSize=36.5 K 2024-11-08T02:23:25,387 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 15e120ae87054579934adbd666208370, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731032598137 2024-11-08T02:23:25,387 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bd0d97cebf54c5ab4bd4fdb5ee0aab4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731032600330 2024-11-08T02:23:25,387 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 125363c13d9b4cbf9c867d25776f7167, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731032602543 2024-11-08T02:23:25,400 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ecb51078105f519ce4baa47714771b2#C#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:25,401 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/759949c8ee084d6ba163b418bc617c1b is 50, key is test_row_0/C:col10/1731032602543/Put/seqid=0 2024-11-08T02:23:25,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742053_1229 (size=13017) 2024-11-08T02:23:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:25,429 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:25,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T02:23:25,430 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-11-08T02:23:25,430 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:25,430 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:25,430 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:25,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:25,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fd30363e7d514cd8ae17eacfb878242f_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_0/A:col10/1731032603242/Put/seqid=0 2024-11-08T02:23:25,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742054_1230 (size=12454) 2024-11-08T02:23:25,463 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/9d098a6b9e3948408f1ea7e518c1f060 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/9d098a6b9e3948408f1ea7e518c1f060 2024-11-08T02:23:25,468 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/A of 0ecb51078105f519ce4baa47714771b2 into 9d098a6b9e3948408f1ea7e518c1f060(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:25,468 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:25,469 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/A, priority=13, startTime=1731032604909; duration=0sec 2024-11-08T02:23:25,469 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:25,469 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:A 2024-11-08T02:23:25,815 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/759949c8ee084d6ba163b418bc617c1b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/759949c8ee084d6ba163b418bc617c1b 2024-11-08T02:23:25,822 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ecb51078105f519ce4baa47714771b2/C of 0ecb51078105f519ce4baa47714771b2 into 759949c8ee084d6ba163b418bc617c1b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:25,822 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:25,822 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., storeName=0ecb51078105f519ce4baa47714771b2/C, priority=13, startTime=1731032604910; duration=0sec 2024-11-08T02:23:25,822 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:25,822 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ecb51078105f519ce4baa47714771b2:C 2024-11-08T02:23:25,843 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:25,848 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fd30363e7d514cd8ae17eacfb878242f_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fd30363e7d514cd8ae17eacfb878242f_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:25,849 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/20562bce76b34f138d5a2a96f75524c9, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:25,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/20562bce76b34f138d5a2a96f75524c9 is 175, key is test_row_0/A:col10/1731032603242/Put/seqid=0 2024-11-08T02:23:25,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742055_1231 (size=31255) 2024-11-08T02:23:26,253 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/20562bce76b34f138d5a2a96f75524c9 2024-11-08T02:23:26,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/cd26cfea576343b6a9192c395abf3f4a is 50, key is test_row_0/B:col10/1731032603242/Put/seqid=0 2024-11-08T02:23:26,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742056_1232 (size=12301) 2024-11-08T02:23:26,569 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:23:26,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/cd26cfea576343b6a9192c395abf3f4a 2024-11-08T02:23:26,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/a5b57d6e28d14e7aa47abcd8bf879a2d is 50, key is test_row_0/C:col10/1731032603242/Put/seqid=0 2024-11-08T02:23:26,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742057_1233 (size=12301) 2024-11-08T02:23:27,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/a5b57d6e28d14e7aa47abcd8bf879a2d 2024-11-08T02:23:27,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/20562bce76b34f138d5a2a96f75524c9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/20562bce76b34f138d5a2a96f75524c9 2024-11-08T02:23:27,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/20562bce76b34f138d5a2a96f75524c9, entries=150, sequenceid=312, filesize=30.5 K 2024-11-08T02:23:27,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/cd26cfea576343b6a9192c395abf3f4a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/cd26cfea576343b6a9192c395abf3f4a 2024-11-08T02:23:27,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/cd26cfea576343b6a9192c395abf3f4a, entries=150, sequenceid=312, filesize=12.0 K 2024-11-08T02:23:27,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/a5b57d6e28d14e7aa47abcd8bf879a2d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/a5b57d6e28d14e7aa47abcd8bf879a2d 2024-11-08T02:23:27,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/a5b57d6e28d14e7aa47abcd8bf879a2d, entries=150, sequenceid=312, filesize=12.0 K 2024-11-08T02:23:27,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 0ecb51078105f519ce4baa47714771b2 in 1666ms, sequenceid=312, compaction requested=false 2024-11-08T02:23:27,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ecb51078105f519ce4baa47714771b2: 2024-11-08T02:23:33,530 INFO [Thread-701 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-701. 2024-11-08T02:23:33,530 DEBUG [Thread-701 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:33,530 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:33,530 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:33,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:33,549 INFO [Thread-697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-697. 2024-11-08T02:23:33,550 DEBUG [Thread-697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:33,550 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:33,550 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:33,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:33,570 INFO [Thread-699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-699. 2024-11-08T02:23:33,570 DEBUG [Thread-699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:33,570 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:33,570 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:33,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:33,589 INFO [Thread-703 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-703. 2024-11-08T02:23:33,589 DEBUG [Thread-703 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:33,589 DEBUG [Thread-703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:33,589 DEBUG [Thread-703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:33,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 207 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4225 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4313 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1832 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5492 rows 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1830 2024-11-08T02:23:33,590 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5486 rows 2024-11-08T02:23:33,590 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T02:23:33,590 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:23:33,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:33,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:33,590 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:33,591 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:23:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T02:23:33,594 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032613594"}]},"ts":"1731032613594"} 2024-11-08T02:23:33,596 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:23:33,596 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:23:33,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:23:33,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, UNASSIGN}] 2024-11-08T02:23:33,598 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, UNASSIGN 2024-11-08T02:23:33,599 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=0ecb51078105f519ce4baa47714771b2, regionState=CLOSING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:23:33,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, UNASSIGN because future has completed 2024-11-08T02:23:33,600 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:23:33,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:23:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T02:23:33,753 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:33,753 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T02:23:33,753 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 0ecb51078105f519ce4baa47714771b2, disabling compactions & flushes 2024-11-08T02:23:33,753 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:33,753 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:33,753 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. after waiting 0 ms 2024-11-08T02:23:33,753 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:33,753 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 0ecb51078105f519ce4baa47714771b2 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T02:23:33,760 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080ef18a9456174a4a85ad72224ec7861d_0ecb51078105f519ce4baa47714771b2 is 50, key is test_row_1/A:col10/1731032613568/Put/seqid=0 2024-11-08T02:23:33,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742058_1234 (size=9914) 2024-11-08T02:23:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T02:23:34,165 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:34,170 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080ef18a9456174a4a85ad72224ec7861d_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080ef18a9456174a4a85ad72224ec7861d_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:34,171 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/3a5cf24b299846a785dbaa36c7d59a1c, store: [table=TestAcidGuarantees family=A region=0ecb51078105f519ce4baa47714771b2] 2024-11-08T02:23:34,172 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/3a5cf24b299846a785dbaa36c7d59a1c is 175, key is test_row_1/A:col10/1731032613568/Put/seqid=0 2024-11-08T02:23:34,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742059_1235 (size=22561) 2024-11-08T02:23:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T02:23:34,576 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/3a5cf24b299846a785dbaa36c7d59a1c 2024-11-08T02:23:34,584 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/ee68d8206c9d45d280a79b7c3ed8e804 is 50, key is test_row_1/B:col10/1731032613568/Put/seqid=0 2024-11-08T02:23:34,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742060_1236 (size=9857) 2024-11-08T02:23:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T02:23:34,988 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/ee68d8206c9d45d280a79b7c3ed8e804 2024-11-08T02:23:34,995 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/70b4919d4059447a98984e3079984dde is 50, key is test_row_1/C:col10/1731032613568/Put/seqid=0 2024-11-08T02:23:34,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742061_1237 (size=9857) 2024-11-08T02:23:35,399 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/70b4919d4059447a98984e3079984dde 2024-11-08T02:23:35,404 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/A/3a5cf24b299846a785dbaa36c7d59a1c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/3a5cf24b299846a785dbaa36c7d59a1c 2024-11-08T02:23:35,408 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/3a5cf24b299846a785dbaa36c7d59a1c, entries=100, sequenceid=321, filesize=22.0 K 2024-11-08T02:23:35,409 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/B/ee68d8206c9d45d280a79b7c3ed8e804 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ee68d8206c9d45d280a79b7c3ed8e804 2024-11-08T02:23:35,413 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ee68d8206c9d45d280a79b7c3ed8e804, entries=100, sequenceid=321, filesize=9.6 K 2024-11-08T02:23:35,414 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/.tmp/C/70b4919d4059447a98984e3079984dde as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/70b4919d4059447a98984e3079984dde 2024-11-08T02:23:35,418 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/70b4919d4059447a98984e3079984dde, entries=100, sequenceid=321, filesize=9.6 K 2024-11-08T02:23:35,419 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0ecb51078105f519ce4baa47714771b2 in 1666ms, sequenceid=321, compaction requested=true 2024-11-08T02:23:35,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f82641e8ddb4492d88ec1c3ee91c9b44, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c97b19edbe4a4c808663563b493f1512, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/19f01ddaae60428fb09562252356d7a9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/84868687bd1340238b318d44bbf427c4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/6fbcb5c3d641475aa13d9b393e7f220d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/65218c244c80494c889ae033e34c66d4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/0257d2c641c54c4ea218ca76ada5ba03, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/527148a7b7ed435b975de29dcc3599a6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c356c0d2d16f48c1a46f285a8657ca11, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5fb98af47044414598ef421762e922d1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/25404ecf405c4dfa803d9c27579e5405, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5963ff1f9ae1477e90aceb1a452b2c07, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/ce1ab94f2a5a4f1f98c03bd5d88a5865, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/29c28ef0505b4616833e39314fff27a9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/d0e7bb55fd8542698c3a692969045e30, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/afdf9aae7ef14450bba9a929ab35bd42, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/72d00e7c19cf465b9dc178d2cf031290, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/456c153b68e44942b8f18dc5542c936c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f4621923375948b8b8730b85d89e90b8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f1334b58fde14e738685967e34acb69c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/35c758025ae64274992bf4b7f5c2b593] to archive 2024-11-08T02:23:35,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:23:35,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f82641e8ddb4492d88ec1c3ee91c9b44 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f82641e8ddb4492d88ec1c3ee91c9b44 2024-11-08T02:23:35,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c97b19edbe4a4c808663563b493f1512 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c97b19edbe4a4c808663563b493f1512 2024-11-08T02:23:35,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/19f01ddaae60428fb09562252356d7a9 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/19f01ddaae60428fb09562252356d7a9 2024-11-08T02:23:35,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/84868687bd1340238b318d44bbf427c4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/84868687bd1340238b318d44bbf427c4 2024-11-08T02:23:35,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/6fbcb5c3d641475aa13d9b393e7f220d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/6fbcb5c3d641475aa13d9b393e7f220d 2024-11-08T02:23:35,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/65218c244c80494c889ae033e34c66d4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/65218c244c80494c889ae033e34c66d4 2024-11-08T02:23:35,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/0257d2c641c54c4ea218ca76ada5ba03 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/0257d2c641c54c4ea218ca76ada5ba03 2024-11-08T02:23:35,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/527148a7b7ed435b975de29dcc3599a6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/527148a7b7ed435b975de29dcc3599a6 2024-11-08T02:23:35,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c356c0d2d16f48c1a46f285a8657ca11 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/c356c0d2d16f48c1a46f285a8657ca11 2024-11-08T02:23:35,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5fb98af47044414598ef421762e922d1 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5fb98af47044414598ef421762e922d1 2024-11-08T02:23:35,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/25404ecf405c4dfa803d9c27579e5405 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/25404ecf405c4dfa803d9c27579e5405 2024-11-08T02:23:35,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5963ff1f9ae1477e90aceb1a452b2c07 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/5963ff1f9ae1477e90aceb1a452b2c07 2024-11-08T02:23:35,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/ce1ab94f2a5a4f1f98c03bd5d88a5865 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/ce1ab94f2a5a4f1f98c03bd5d88a5865 2024-11-08T02:23:35,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/29c28ef0505b4616833e39314fff27a9 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/29c28ef0505b4616833e39314fff27a9 2024-11-08T02:23:35,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/d0e7bb55fd8542698c3a692969045e30 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/d0e7bb55fd8542698c3a692969045e30 2024-11-08T02:23:35,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/afdf9aae7ef14450bba9a929ab35bd42 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/afdf9aae7ef14450bba9a929ab35bd42 2024-11-08T02:23:35,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/72d00e7c19cf465b9dc178d2cf031290 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/72d00e7c19cf465b9dc178d2cf031290 2024-11-08T02:23:35,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/456c153b68e44942b8f18dc5542c936c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/456c153b68e44942b8f18dc5542c936c 2024-11-08T02:23:35,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f4621923375948b8b8730b85d89e90b8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f4621923375948b8b8730b85d89e90b8 2024-11-08T02:23:35,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f1334b58fde14e738685967e34acb69c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/f1334b58fde14e738685967e34acb69c 2024-11-08T02:23:35,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/35c758025ae64274992bf4b7f5c2b593 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/35c758025ae64274992bf4b7f5c2b593 2024-11-08T02:23:35,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/3c62cfd5a5bb4e068743f252d10b88be, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/fa4f5102a92447a3a85bede21032361b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/3ddd441331d543938eb92ac6b212406d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/bd99e9548c904895884c00f5b440d6f3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6b8e791cf68844f1b0cda1b0f338fd3f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1965fc4db1c64016acd9043ec5479364, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e19552a2f3c24b3791c953cbd7bd38e8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/515297aff9f24da8859fdaa7498379ec, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/47f20814eb85463789488526eec993df, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/2200a056b078476587df54443bbf9c46, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/52142225e6cd4c0b95b13bfc47b2dab9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/106899eed2d646baaa7c82d2ab1ffca0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6a5186dfc9a14e979a032ca2fb93d780, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/62c5de302df64e17870f7e637faef5ca, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/9301fead6acb4f7587e318e2898e0150, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/143bd6d81944432c8220fec1706759b5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ad90b6ea198a4b5d80f7111e15d87355, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/23d278d8c6f7495e926e4d9099517535, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ceaea4875c564d789eaa9d24bca58300, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1832b53f6ac345f5a3ad29ebea0a2858, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e9f75bac95cc4da387359a679f816eee] to archive 2024-11-08T02:23:35,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:23:35,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/3c62cfd5a5bb4e068743f252d10b88be to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/3c62cfd5a5bb4e068743f252d10b88be 2024-11-08T02:23:35,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/fa4f5102a92447a3a85bede21032361b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/fa4f5102a92447a3a85bede21032361b 2024-11-08T02:23:35,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/3ddd441331d543938eb92ac6b212406d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/3ddd441331d543938eb92ac6b212406d 2024-11-08T02:23:35,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/bd99e9548c904895884c00f5b440d6f3 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/bd99e9548c904895884c00f5b440d6f3 2024-11-08T02:23:35,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6b8e791cf68844f1b0cda1b0f338fd3f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6b8e791cf68844f1b0cda1b0f338fd3f 2024-11-08T02:23:35,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1965fc4db1c64016acd9043ec5479364 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1965fc4db1c64016acd9043ec5479364 2024-11-08T02:23:35,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e19552a2f3c24b3791c953cbd7bd38e8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e19552a2f3c24b3791c953cbd7bd38e8 2024-11-08T02:23:35,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/515297aff9f24da8859fdaa7498379ec to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/515297aff9f24da8859fdaa7498379ec 2024-11-08T02:23:35,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/47f20814eb85463789488526eec993df to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/47f20814eb85463789488526eec993df 2024-11-08T02:23:35,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/2200a056b078476587df54443bbf9c46 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/2200a056b078476587df54443bbf9c46 2024-11-08T02:23:35,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/52142225e6cd4c0b95b13bfc47b2dab9 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/52142225e6cd4c0b95b13bfc47b2dab9 2024-11-08T02:23:35,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/106899eed2d646baaa7c82d2ab1ffca0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/106899eed2d646baaa7c82d2ab1ffca0 2024-11-08T02:23:35,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6a5186dfc9a14e979a032ca2fb93d780 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/6a5186dfc9a14e979a032ca2fb93d780 2024-11-08T02:23:35,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/62c5de302df64e17870f7e637faef5ca to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/62c5de302df64e17870f7e637faef5ca 2024-11-08T02:23:35,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/9301fead6acb4f7587e318e2898e0150 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/9301fead6acb4f7587e318e2898e0150 2024-11-08T02:23:35,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/143bd6d81944432c8220fec1706759b5 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/143bd6d81944432c8220fec1706759b5 2024-11-08T02:23:35,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ad90b6ea198a4b5d80f7111e15d87355 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ad90b6ea198a4b5d80f7111e15d87355 2024-11-08T02:23:35,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/23d278d8c6f7495e926e4d9099517535 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/23d278d8c6f7495e926e4d9099517535 2024-11-08T02:23:35,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ceaea4875c564d789eaa9d24bca58300 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ceaea4875c564d789eaa9d24bca58300 2024-11-08T02:23:35,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1832b53f6ac345f5a3ad29ebea0a2858 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1832b53f6ac345f5a3ad29ebea0a2858 2024-11-08T02:23:35,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e9f75bac95cc4da387359a679f816eee to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/e9f75bac95cc4da387359a679f816eee 2024-11-08T02:23:35,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ff868e9a25bc430686bf59ea1e816cc7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/b328bab3893a422bb452ca050f5c0d5d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/dbfbbef33cf54c5497137e46d574f845, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/3431d6afdd1d45189fc389e29b27f760, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d3e6e8499ec64766b758969d9a42a611, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/4d5495af73b545fb887ae37459244334, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/252831ea77bd41a5802dfb7a37c03aa4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ea77ad4937874145af048171939db707, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/069f30ec9edd4f9185056fc8bf3ddc80, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/07e246e21c0e4478bb5a326c032e7928, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/430a06a1b82749f7983061769c57f85c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/cd87c82ac6104b189f5277d5e2519f0f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/2bdd73c4c8c6496eab79b5353bed3319, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/60a100524734461e82ba59bf73ac428b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/6ca4e11074fb4d6eaad3aee8d99ccaaa, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/fc05a157dc174008bfd8e9a8e03d11ed, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d14246d9ac654e64a899df68511b09ae, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/15e120ae87054579934adbd666208370, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/09d687f0c8644ce9869f9fa4add73a04, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/9bd0d97cebf54c5ab4bd4fdb5ee0aab4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/125363c13d9b4cbf9c867d25776f7167] to archive 2024-11-08T02:23:35,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:23:35,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ff868e9a25bc430686bf59ea1e816cc7 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ff868e9a25bc430686bf59ea1e816cc7 2024-11-08T02:23:35,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/b328bab3893a422bb452ca050f5c0d5d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/b328bab3893a422bb452ca050f5c0d5d 2024-11-08T02:23:35,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/dbfbbef33cf54c5497137e46d574f845 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/dbfbbef33cf54c5497137e46d574f845 2024-11-08T02:23:35,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/3431d6afdd1d45189fc389e29b27f760 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/3431d6afdd1d45189fc389e29b27f760 2024-11-08T02:23:35,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d3e6e8499ec64766b758969d9a42a611 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d3e6e8499ec64766b758969d9a42a611 2024-11-08T02:23:35,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/4d5495af73b545fb887ae37459244334 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/4d5495af73b545fb887ae37459244334 2024-11-08T02:23:35,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/252831ea77bd41a5802dfb7a37c03aa4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/252831ea77bd41a5802dfb7a37c03aa4 2024-11-08T02:23:35,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ea77ad4937874145af048171939db707 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/ea77ad4937874145af048171939db707 2024-11-08T02:23:35,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/069f30ec9edd4f9185056fc8bf3ddc80 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/069f30ec9edd4f9185056fc8bf3ddc80 2024-11-08T02:23:35,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/07e246e21c0e4478bb5a326c032e7928 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/07e246e21c0e4478bb5a326c032e7928 2024-11-08T02:23:35,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/430a06a1b82749f7983061769c57f85c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/430a06a1b82749f7983061769c57f85c 2024-11-08T02:23:35,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/cd87c82ac6104b189f5277d5e2519f0f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/cd87c82ac6104b189f5277d5e2519f0f 2024-11-08T02:23:35,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/2bdd73c4c8c6496eab79b5353bed3319 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/2bdd73c4c8c6496eab79b5353bed3319 2024-11-08T02:23:35,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/60a100524734461e82ba59bf73ac428b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/60a100524734461e82ba59bf73ac428b 2024-11-08T02:23:35,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/6ca4e11074fb4d6eaad3aee8d99ccaaa to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/6ca4e11074fb4d6eaad3aee8d99ccaaa 2024-11-08T02:23:35,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/fc05a157dc174008bfd8e9a8e03d11ed to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/fc05a157dc174008bfd8e9a8e03d11ed 2024-11-08T02:23:35,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d14246d9ac654e64a899df68511b09ae to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/d14246d9ac654e64a899df68511b09ae 2024-11-08T02:23:35,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/15e120ae87054579934adbd666208370 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/15e120ae87054579934adbd666208370 2024-11-08T02:23:35,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/09d687f0c8644ce9869f9fa4add73a04 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/09d687f0c8644ce9869f9fa4add73a04 2024-11-08T02:23:35,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/9bd0d97cebf54c5ab4bd4fdb5ee0aab4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/9bd0d97cebf54c5ab4bd4fdb5ee0aab4 2024-11-08T02:23:35,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/125363c13d9b4cbf9c867d25776f7167 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/125363c13d9b4cbf9c867d25776f7167 2024-11-08T02:23:35,491 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/recovered.edits/324.seqid, newMaxSeqId=324, maxSeqId=4 2024-11-08T02:23:35,491 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. 2024-11-08T02:23:35,491 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 0ecb51078105f519ce4baa47714771b2: Waiting for close lock at 1731032613753Running coprocessor pre-close hooks at 1731032613753Disabling compacts and flushes for region at 1731032613753Disabling writes for close at 1731032613753Obtaining lock to block concurrent updates at 1731032613753Preparing flush snapshotting stores in 0ecb51078105f519ce4baa47714771b2 at 1731032613753Finished memstore snapshotting TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731032613754 (+1 ms)Flushing stores of TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2. at 1731032613754Flushing 0ecb51078105f519ce4baa47714771b2/A: creating writer at 1731032613754Flushing mob file 0ecb51078105f519ce4baa47714771b2/A: appending metadata at 1731032613760 (+6 ms)Flushing mob file 0ecb51078105f519ce4baa47714771b2/A: closing flushed file at 1731032613760Flushing 0ecb51078105f519ce4baa47714771b2/A: appending metadata at 1731032614171 (+411 ms)Flushing 0ecb51078105f519ce4baa47714771b2/A: closing flushed file at 1731032614171Flushing 0ecb51078105f519ce4baa47714771b2/B: creating writer at 1731032614580 (+409 ms)Flushing 0ecb51078105f519ce4baa47714771b2/B: appending metadata at 1731032614583 (+3 ms)Flushing 0ecb51078105f519ce4baa47714771b2/B: closing flushed file at 1731032614583Flushing 0ecb51078105f519ce4baa47714771b2/C: creating writer at 1731032614992 (+409 ms)Flushing 0ecb51078105f519ce4baa47714771b2/C: appending metadata at 1731032614995 (+3 ms)Flushing 0ecb51078105f519ce4baa47714771b2/C: closing flushed file at 1731032614995Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77fe3bf5: reopening flushed file at 1731032615404 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44837b86: reopening flushed file at 1731032615409 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@394c0c4c: reopening flushed file at 1731032615413 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 0ecb51078105f519ce4baa47714771b2 in 1666ms, sequenceid=321, compaction requested=true at 1731032615419 (+6 ms)Writing region close event to WAL at 1731032615488 (+69 ms)Running coprocessor post-close hooks at 1731032615491 (+3 ms)Closed at 1731032615491 2024-11-08T02:23:35,493 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,494 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=0ecb51078105f519ce4baa47714771b2, regionState=CLOSED 2024-11-08T02:23:35,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:23:35,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-08T02:23:35,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 0ecb51078105f519ce4baa47714771b2, server=331c7316141f,34325,1731032548147 in 1.8960 sec 2024-11-08T02:23:35,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-08T02:23:35,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ecb51078105f519ce4baa47714771b2, UNASSIGN in 1.9000 sec 2024-11-08T02:23:35,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-08T02:23:35,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9030 sec 2024-11-08T02:23:35,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032615502"}]},"ts":"1731032615502"} 2024-11-08T02:23:35,503 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:23:35,503 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:23:35,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9130 sec 2024-11-08T02:23:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T02:23:35,738 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:35,738 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:23:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:35,740 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T02:23:35,741 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:35,743 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,746 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/recovered.edits] 2024-11-08T02:23:35,749 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/20562bce76b34f138d5a2a96f75524c9 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/20562bce76b34f138d5a2a96f75524c9 2024-11-08T02:23:35,750 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/3a5cf24b299846a785dbaa36c7d59a1c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/3a5cf24b299846a785dbaa36c7d59a1c 2024-11-08T02:23:35,752 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/9d098a6b9e3948408f1ea7e518c1f060 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/A/9d098a6b9e3948408f1ea7e518c1f060 2024-11-08T02:23:35,754 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1c25fd9df49e4322ae7e0058cd155513 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/1c25fd9df49e4322ae7e0058cd155513 2024-11-08T02:23:35,755 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/cd26cfea576343b6a9192c395abf3f4a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/cd26cfea576343b6a9192c395abf3f4a 2024-11-08T02:23:35,756 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ee68d8206c9d45d280a79b7c3ed8e804 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/B/ee68d8206c9d45d280a79b7c3ed8e804 2024-11-08T02:23:35,758 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/70b4919d4059447a98984e3079984dde to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/70b4919d4059447a98984e3079984dde 2024-11-08T02:23:35,759 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/759949c8ee084d6ba163b418bc617c1b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/759949c8ee084d6ba163b418bc617c1b 2024-11-08T02:23:35,760 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/a5b57d6e28d14e7aa47abcd8bf879a2d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/C/a5b57d6e28d14e7aa47abcd8bf879a2d 2024-11-08T02:23:35,763 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/recovered.edits/324.seqid to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2/recovered.edits/324.seqid 2024-11-08T02:23:35,763 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,763 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:23:35,764 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:23:35,765 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T02:23:35,769 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108049b4896bc514e41ab2f0b45e48c2476_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108049b4896bc514e41ab2f0b45e48c2476_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,770 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080ef18a9456174a4a85ad72224ec7861d_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080ef18a9456174a4a85ad72224ec7861d_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,771 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110814c236e72809427185721e5e47d8844a_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110814c236e72809427185721e5e47d8844a_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,772 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081a957cca1c2e41b690885f0aa72131aa_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081a957cca1c2e41b690885f0aa72131aa_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,773 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108325a6eb6ebf84e33ab64273ac0294aa0_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108325a6eb6ebf84e33ab64273ac0294aa0_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,775 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083be390d128e244c6a93cb6b6b71204b5_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083be390d128e244c6a93cb6b6b71204b5_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,776 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083e221f9b408c41018b784d7c93e454ee_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083e221f9b408c41018b784d7c93e454ee_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,777 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085b6ed291bbcb4ac4aac66b248c06684b_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085b6ed291bbcb4ac4aac66b248c06684b_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,779 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085fd47e845d8a4190a484d15302d912fd_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085fd47e845d8a4190a484d15302d912fd_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,780 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108693449eea26c43fdab5abaa6bd310fea_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108693449eea26c43fdab5abaa6bd310fea_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,781 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089bdf34172da3430cb96c9f9922553c48_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089bdf34172da3430cb96c9f9922553c48_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,782 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a54fc624904446b9842c9572f33beeee_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a54fc624904446b9842c9572f33beeee_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,783 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cfe95ab77c57496d916bd63bb399a3f4_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cfe95ab77c57496d916bd63bb399a3f4_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,784 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ea75596df68a4cfcb61223cdceb3b676_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ea75596df68a4cfcb61223cdceb3b676_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,785 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ecd38f4a5ea44f2983fefb02d71ede3f_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ecd38f4a5ea44f2983fefb02d71ede3f_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,787 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fb5770b256b34f7bb40dd47c8787b4f0_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fb5770b256b34f7bb40dd47c8787b4f0_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,787 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fd30363e7d514cd8ae17eacfb878242f_0ecb51078105f519ce4baa47714771b2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fd30363e7d514cd8ae17eacfb878242f_0ecb51078105f519ce4baa47714771b2 2024-11-08T02:23:35,788 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:23:35,790 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:35,793 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:23:35,795 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:23:35,796 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:35,796 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:23:35,796 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731032615796"}]},"ts":"9223372036854775807"} 2024-11-08T02:23:35,798 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T02:23:35,798 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0ecb51078105f519ce4baa47714771b2, NAME => 'TestAcidGuarantees,,1731032580769.0ecb51078105f519ce4baa47714771b2.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:23:35,798 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:23:35,799 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731032615799"}]},"ts":"9223372036854775807"} 2024-11-08T02:23:35,801 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T02:23:35,802 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:35,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-08T02:23:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T02:23:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T02:23:35,848 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:35,861 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 238), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=510 (was 550), ProcessCount=11 (was 11), AvailableMemoryMB=6818 (was 7058) 2024-11-08T02:23:35,871 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=510, ProcessCount=11, AvailableMemoryMB=6815 2024-11-08T02:23:35,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:23:35,873 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:23:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:23:35,875 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:23:35,875 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:35,875 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-11-08T02:23:35,876 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:23:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-08T02:23:35,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742062_1238 (size=959) 2024-11-08T02:23:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-08T02:23:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-08T02:23:36,284 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee 2024-11-08T02:23:36,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742063_1239 (size=53) 2024-11-08T02:23:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-08T02:23:36,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:23:36,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1dba34e1c528fab2e73d505d0e433dae, disabling compactions & flushes 2024-11-08T02:23:36,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:36,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:36,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. after waiting 0 ms 2024-11-08T02:23:36,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:36,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:36,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1dba34e1c528fab2e73d505d0e433dae: Waiting for close lock at 1731032616698Disabling compacts and flushes for region at 1731032616698Disabling writes for close at 1731032616698Writing region close event to WAL at 1731032616698Closed at 1731032616698 2024-11-08T02:23:36,700 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:23:36,700 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731032616700"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731032616700"}]},"ts":"1731032616700"} 2024-11-08T02:23:36,702 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T02:23:36,703 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:23:36,703 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032616703"}]},"ts":"1731032616703"} 2024-11-08T02:23:36,704 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:23:36,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1dba34e1c528fab2e73d505d0e433dae, ASSIGN}] 2024-11-08T02:23:36,706 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1dba34e1c528fab2e73d505d0e433dae, ASSIGN 2024-11-08T02:23:36,706 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1dba34e1c528fab2e73d505d0e433dae, ASSIGN; state=OFFLINE, location=331c7316141f,34325,1731032548147; forceNewPlan=false, retain=false 2024-11-08T02:23:36,857 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=1dba34e1c528fab2e73d505d0e433dae, regionState=OPENING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:23:36,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1dba34e1c528fab2e73d505d0e433dae, ASSIGN because future has completed 2024-11-08T02:23:36,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:23:37,014 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:37,014 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:23:37,014 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,014 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:23:37,014 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,014 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,016 INFO [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,017 INFO [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1dba34e1c528fab2e73d505d0e433dae columnFamilyName A 2024-11-08T02:23:37,017 DEBUG [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:37,017 INFO [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] regionserver.HStore(327): Store=1dba34e1c528fab2e73d505d0e433dae/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:23:37,018 INFO [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-08T02:23:37,019 INFO [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1dba34e1c528fab2e73d505d0e433dae columnFamilyName B 2024-11-08T02:23:37,019 DEBUG [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:37,019 INFO [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] regionserver.HStore(327): Store=1dba34e1c528fab2e73d505d0e433dae/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:23:37,019 INFO [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,020 INFO [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1dba34e1c528fab2e73d505d0e433dae columnFamilyName C 2024-11-08T02:23:37,020 DEBUG [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:23:37,021 INFO [StoreOpener-1dba34e1c528fab2e73d505d0e433dae-1 {}] regionserver.HStore(327): Store=1dba34e1c528fab2e73d505d0e433dae/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:23:37,021 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,022 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,022 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,023 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,023 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,023 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:23:37,024 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,026 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:23:37,026 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened 1dba34e1c528fab2e73d505d0e433dae; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66553288, jitterRate=-0.008278727531433105}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:23:37,026 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:37,027 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for 1dba34e1c528fab2e73d505d0e433dae: Running coprocessor pre-open hook at 1731032617014Writing region info on filesystem at 1731032617014Initializing all the Stores at 1731032617015 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032617015Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032617015Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032617015Cleaning up temporary data from old regions at 1731032617023 (+8 ms)Running coprocessor post-open hooks at 1731032617026 (+3 ms)Region opened successfully at 1731032617027 (+1 ms) 2024-11-08T02:23:37,028 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., pid=57, masterSystemTime=1731032617011 2024-11-08T02:23:37,029 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:37,029 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:37,030 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=1dba34e1c528fab2e73d505d0e433dae, regionState=OPEN, openSeqNum=2, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:23:37,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:23:37,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-08T02:23:37,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure 1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 in 173 msec 2024-11-08T02:23:37,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-08T02:23:37,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1dba34e1c528fab2e73d505d0e433dae, ASSIGN in 328 msec 2024-11-08T02:23:37,036 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:23:37,036 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032617036"}]},"ts":"1731032617036"} 2024-11-08T02:23:37,037 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:23:37,038 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:23:37,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-08T02:23:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-11-08T02:23:38,028 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:38,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:38,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:38,030 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c031d5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:38,032 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:38,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405509c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,032 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:38,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:38,035 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T02:23:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 331c7316141f,41605,1731032547397 2024-11-08T02:23:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c4a80f0 2024-11-08T02:23:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:23:38,038 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:23:38,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:38,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:38,040 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:38,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:38,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:38,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@457f8ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:38,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:38,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:38,042 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:38,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@419b1f80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:38,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:38,044 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a6e29a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:38,045 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383d7715, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,047 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:38,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:38,047 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618a887b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:38,049 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:38,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aec7c4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:38,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:38,051 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:38,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:38,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:38,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65455877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:38,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:38,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:38,052 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f254f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:38,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:38,054 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef8aa34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:38,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:38,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:38,056 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:38,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1671c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:38,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:38,057 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@345e841d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:38,059 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6077d7ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:38,060 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:38,060 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:38,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64e2db3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:38,061 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ebeca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,063 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:38,063 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:38,063 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17c3355f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:38,064 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5186f825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,066 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:38,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:38,066 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:38,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52eed675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:38,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:38,068 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@321f6d3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:23:38,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:23:38,069 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:23:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:23:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:23:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aadfe98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:23:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:23:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:38,070 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:23:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eef3f86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:23:38,071 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:38,071 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:38,072 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:38,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:38,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:38,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:38,072 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:38,072 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:38,072 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:38,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:38,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:38,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:38,073 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-08T02:23:38,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:38,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:38,074 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:38,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:38,075 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:38,075 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:38,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:38,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-08T02:23:38,076 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:38,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:23:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:23:38,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:23:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:23:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:38,079 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:38,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:23:38,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:23:38,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:23:38,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:23:38,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:23:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:38,091 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:38,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:23:38,092 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:23:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:23:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:23:38,094 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:23:38,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:23:38,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/df4c709337224cd3a829579559b53444 is 50, key is test_row_0/A:col10/1731032618089/Put/seqid=0 2024-11-08T02:23:38,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032678124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032678127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032678130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032678132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032678132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742064_1240 (size=12001) 2024-11-08T02:23:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-08T02:23:38,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-08T02:23:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:38,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032678238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032678238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032678238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032678247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032678248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-08T02:23:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:38,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:38,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-08T02:23:38,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032678448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032678449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032678451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032678458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032678458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-08T02:23:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:38,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/df4c709337224cd3a829579559b53444 2024-11-08T02:23:38,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/dee419b3140a480087326dae0f1d84f6 is 50, key is test_row_0/B:col10/1731032618089/Put/seqid=0 2024-11-08T02:23:38,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742065_1241 (size=12001) 2024-11-08T02:23:38,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/dee419b3140a480087326dae0f1d84f6 2024-11-08T02:23:38,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/657ce7b4959446888c64af5a9b922efe is 50, key is test_row_0/C:col10/1731032618089/Put/seqid=0 2024-11-08T02:23:38,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742066_1242 (size=12001) 2024-11-08T02:23:38,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/657ce7b4959446888c64af5a9b922efe 2024-11-08T02:23:38,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/df4c709337224cd3a829579559b53444 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/df4c709337224cd3a829579559b53444 2024-11-08T02:23:38,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/df4c709337224cd3a829579559b53444, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T02:23:38,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/dee419b3140a480087326dae0f1d84f6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dee419b3140a480087326dae0f1d84f6 2024-11-08T02:23:38,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-08T02:23:38,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:38,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:38,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:38,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dee419b3140a480087326dae0f1d84f6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T02:23:38,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/657ce7b4959446888c64af5a9b922efe as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/657ce7b4959446888c64af5a9b922efe 2024-11-08T02:23:38,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/657ce7b4959446888c64af5a9b922efe, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T02:23:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-08T02:23:38,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1dba34e1c528fab2e73d505d0e433dae in 618ms, sequenceid=13, compaction requested=false 2024-11-08T02:23:38,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:38,760 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:38,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:23:38,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/085460b9408f4da39cc6cc42e09b572c is 50, key is test_row_0/A:col10/1731032618129/Put/seqid=0 2024-11-08T02:23:38,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032678769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032678770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032678773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032678774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032678775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742067_1243 (size=14341) 2024-11-08T02:23:38,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/085460b9408f4da39cc6cc42e09b572c 2024-11-08T02:23:38,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/5f9856fcdb9a4a0fa4d4b205ed56b241 is 50, key is test_row_0/B:col10/1731032618129/Put/seqid=0 2024-11-08T02:23:38,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742068_1244 (size=12001) 2024-11-08T02:23:38,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/5f9856fcdb9a4a0fa4d4b205ed56b241 2024-11-08T02:23:38,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-08T02:23:38,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:38,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:38,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:38,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:38,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/9d003ced8bf543b189cc0c28c5efda39 is 50, key is test_row_0/C:col10/1731032618129/Put/seqid=0 2024-11-08T02:23:38,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032678878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032678878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742069_1245 (size=12001) 2024-11-08T02:23:38,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/9d003ced8bf543b189cc0c28c5efda39 2024-11-08T02:23:38,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032678888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032678888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032678888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:38,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:38,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:38,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/085460b9408f4da39cc6cc42e09b572c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/085460b9408f4da39cc6cc42e09b572c 2024-11-08T02:23:38,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/085460b9408f4da39cc6cc42e09b572c, entries=200, sequenceid=39, filesize=14.0 K 2024-11-08T02:23:38,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/5f9856fcdb9a4a0fa4d4b205ed56b241 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5f9856fcdb9a4a0fa4d4b205ed56b241 2024-11-08T02:23:38,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5f9856fcdb9a4a0fa4d4b205ed56b241, entries=150, sequenceid=39, filesize=11.7 K 2024-11-08T02:23:38,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/9d003ced8bf543b189cc0c28c5efda39 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/9d003ced8bf543b189cc0c28c5efda39 2024-11-08T02:23:38,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/9d003ced8bf543b189cc0c28c5efda39, entries=150, sequenceid=39, filesize=11.7 K 2024-11-08T02:23:38,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 1dba34e1c528fab2e73d505d0e433dae in 162ms, sequenceid=39, compaction requested=false 2024-11-08T02:23:38,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:39,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-08T02:23:39,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:39,000 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:23:39,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/fad699a41d934a309ea2e6cbaf8b6e64 is 50, key is test_row_0/A:col10/1731032618773/Put/seqid=0 2024-11-08T02:23:39,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742070_1246 (size=12001) 2024-11-08T02:23:39,037 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/fad699a41d934a309ea2e6cbaf8b6e64 2024-11-08T02:23:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/572343c2841f4a4d9e33390056d2f4bb is 50, key is test_row_0/B:col10/1731032618773/Put/seqid=0 2024-11-08T02:23:39,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742071_1247 (size=12001) 2024-11-08T02:23:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:39,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:39,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032679125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032679126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032679127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032679132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032679133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-08T02:23:39,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032679238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032679238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032679238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032679248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032679248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032679448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032679448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032679456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032679457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032679458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,468 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/572343c2841f4a4d9e33390056d2f4bb 2024-11-08T02:23:39,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/7eca0758584d43d0ae5316c118e21d2a is 50, key is test_row_0/C:col10/1731032618773/Put/seqid=0 2024-11-08T02:23:39,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742072_1248 (size=12001) 2024-11-08T02:23:39,496 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/7eca0758584d43d0ae5316c118e21d2a 2024-11-08T02:23:39,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/fad699a41d934a309ea2e6cbaf8b6e64 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/fad699a41d934a309ea2e6cbaf8b6e64 2024-11-08T02:23:39,507 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/fad699a41d934a309ea2e6cbaf8b6e64, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T02:23:39,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/572343c2841f4a4d9e33390056d2f4bb as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/572343c2841f4a4d9e33390056d2f4bb 2024-11-08T02:23:39,514 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/572343c2841f4a4d9e33390056d2f4bb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T02:23:39,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/7eca0758584d43d0ae5316c118e21d2a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/7eca0758584d43d0ae5316c118e21d2a 2024-11-08T02:23:39,522 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/7eca0758584d43d0ae5316c118e21d2a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T02:23:39,523 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1dba34e1c528fab2e73d505d0e433dae in 523ms, sequenceid=50, compaction requested=true 2024-11-08T02:23:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-08T02:23:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-08T02:23:39,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-08T02:23:39,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4480 sec 2024-11-08T02:23:39,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.4550 sec 2024-11-08T02:23:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:39,760 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:39,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:23:39,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/f59acb5586c14e0d98dcb27fce5ed93f is 50, key is test_row_0/A:col10/1731032619130/Put/seqid=0 2024-11-08T02:23:39,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032679772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032679773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032679773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032679774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032679774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742073_1249 (size=14341) 2024-11-08T02:23:39,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/f59acb5586c14e0d98dcb27fce5ed93f 2024-11-08T02:23:39,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/48f66f8bd0ab4e9bbcdf116e4fa0927c is 50, key is test_row_0/B:col10/1731032619130/Put/seqid=0 2024-11-08T02:23:39,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742074_1250 (size=12001) 2024-11-08T02:23:39,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032679878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032679888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032679888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032679888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032679888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032680088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032680098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032680098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032680098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032680099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/48f66f8bd0ab4e9bbcdf116e4fa0927c 2024-11-08T02:23:40,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/2daf9bb9ab7240e3bd19c4f755a2f8e3 is 50, key is test_row_0/C:col10/1731032619130/Put/seqid=0 2024-11-08T02:23:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-08T02:23:40,229 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:40,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742075_1251 (size=12001) 2024-11-08T02:23:40,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-08T02:23:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-08T02:23:40,233 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:40,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/2daf9bb9ab7240e3bd19c4f755a2f8e3 2024-11-08T02:23:40,235 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:40,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:40,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/f59acb5586c14e0d98dcb27fce5ed93f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f59acb5586c14e0d98dcb27fce5ed93f 2024-11-08T02:23:40,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f59acb5586c14e0d98dcb27fce5ed93f, entries=200, sequenceid=76, filesize=14.0 K 2024-11-08T02:23:40,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/48f66f8bd0ab4e9bbcdf116e4fa0927c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/48f66f8bd0ab4e9bbcdf116e4fa0927c 2024-11-08T02:23:40,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/48f66f8bd0ab4e9bbcdf116e4fa0927c, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:23:40,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/2daf9bb9ab7240e3bd19c4f755a2f8e3 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/2daf9bb9ab7240e3bd19c4f755a2f8e3 2024-11-08T02:23:40,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/2daf9bb9ab7240e3bd19c4f755a2f8e3, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:23:40,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 1dba34e1c528fab2e73d505d0e433dae in 514ms, sequenceid=76, compaction requested=true 2024-11-08T02:23:40,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:40,275 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:23:40,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:40,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:40,278 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:23:40,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:40,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:40,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:40,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:40,279 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:23:40,279 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/A is initiating minor compaction (all files) 2024-11-08T02:23:40,279 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/A in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:40,279 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/df4c709337224cd3a829579559b53444, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/085460b9408f4da39cc6cc42e09b572c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/fad699a41d934a309ea2e6cbaf8b6e64, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f59acb5586c14e0d98dcb27fce5ed93f] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=51.4 K 2024-11-08T02:23:40,280 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting df4c709337224cd3a829579559b53444, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731032618087 2024-11-08T02:23:40,282 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:23:40,282 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/B is initiating minor compaction (all files) 2024-11-08T02:23:40,282 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/B in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:40,282 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dee419b3140a480087326dae0f1d84f6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5f9856fcdb9a4a0fa4d4b205ed56b241, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/572343c2841f4a4d9e33390056d2f4bb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/48f66f8bd0ab4e9bbcdf116e4fa0927c] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=46.9 K 2024-11-08T02:23:40,282 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 085460b9408f4da39cc6cc42e09b572c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731032618129 2024-11-08T02:23:40,282 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting dee419b3140a480087326dae0f1d84f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731032618087 2024-11-08T02:23:40,283 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting fad699a41d934a309ea2e6cbaf8b6e64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731032618767 2024-11-08T02:23:40,283 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f9856fcdb9a4a0fa4d4b205ed56b241, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731032618129 2024-11-08T02:23:40,284 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 572343c2841f4a4d9e33390056d2f4bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731032618767 2024-11-08T02:23:40,284 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting f59acb5586c14e0d98dcb27fce5ed93f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731032619126 2024-11-08T02:23:40,285 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 48f66f8bd0ab4e9bbcdf116e4fa0927c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731032619126 2024-11-08T02:23:40,305 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#A#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:40,306 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/9977d5e9ee4f44efaf3e8973bb2433cf is 50, key is test_row_0/A:col10/1731032619130/Put/seqid=0 2024-11-08T02:23:40,309 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#B#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:40,310 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/16fd880921264c12b00244cb755f0f34 is 50, key is test_row_0/B:col10/1731032619130/Put/seqid=0 2024-11-08T02:23:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-08T02:23:40,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742076_1252 (size=12139) 2024-11-08T02:23:40,359 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/9977d5e9ee4f44efaf3e8973bb2433cf as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/9977d5e9ee4f44efaf3e8973bb2433cf 2024-11-08T02:23:40,369 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/A of 1dba34e1c528fab2e73d505d0e433dae into 9977d5e9ee4f44efaf3e8973bb2433cf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:40,369 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:40,369 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/A, priority=12, startTime=1731032620275; duration=0sec 2024-11-08T02:23:40,369 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:40,369 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:A 2024-11-08T02:23:40,369 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:23:40,371 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:23:40,371 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/C is initiating minor compaction (all files) 2024-11-08T02:23:40,371 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/C in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:40,371 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/657ce7b4959446888c64af5a9b922efe, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/9d003ced8bf543b189cc0c28c5efda39, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/7eca0758584d43d0ae5316c118e21d2a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/2daf9bb9ab7240e3bd19c4f755a2f8e3] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=46.9 K 2024-11-08T02:23:40,372 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 657ce7b4959446888c64af5a9b922efe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731032618087 2024-11-08T02:23:40,373 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d003ced8bf543b189cc0c28c5efda39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731032618129 2024-11-08T02:23:40,373 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eca0758584d43d0ae5316c118e21d2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731032618767 2024-11-08T02:23:40,375 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2daf9bb9ab7240e3bd19c4f755a2f8e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731032619126 2024-11-08T02:23:40,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742077_1253 (size=12139) 2024-11-08T02:23:40,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-08T02:23:40,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:40,388 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:23:40,392 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/16fd880921264c12b00244cb755f0f34 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/16fd880921264c12b00244cb755f0f34 2024-11-08T02:23:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:40,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:40,400 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/B of 1dba34e1c528fab2e73d505d0e433dae into 16fd880921264c12b00244cb755f0f34(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:40,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/b36fc6ad594b4a98adf65e2fb366b27d is 50, key is test_row_0/A:col10/1731032619772/Put/seqid=0 2024-11-08T02:23:40,400 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:40,403 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/B, priority=12, startTime=1731032620278; duration=0sec 2024-11-08T02:23:40,403 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:40,403 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:B 2024-11-08T02:23:40,415 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#C#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:40,416 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/0ec3c52b2d4d48e6aaaa80ad290f9870 is 50, key is test_row_0/C:col10/1731032619130/Put/seqid=0 2024-11-08T02:23:40,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032680442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032680445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742078_1254 (size=12001) 2024-11-08T02:23:40,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032680448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032680447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,454 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/b36fc6ad594b4a98adf65e2fb366b27d 2024-11-08T02:23:40,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032680457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742079_1255 (size=12139) 2024-11-08T02:23:40,473 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/0ec3c52b2d4d48e6aaaa80ad290f9870 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/0ec3c52b2d4d48e6aaaa80ad290f9870 2024-11-08T02:23:40,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/f4c4eeee599143a1991ef9936b4d1768 is 50, key is test_row_0/B:col10/1731032619772/Put/seqid=0 2024-11-08T02:23:40,510 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/C of 1dba34e1c528fab2e73d505d0e433dae into 0ec3c52b2d4d48e6aaaa80ad290f9870(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:40,510 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:40,510 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/C, priority=12, startTime=1731032620278; duration=0sec 2024-11-08T02:23:40,510 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:40,510 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:C 2024-11-08T02:23:40,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742080_1256 (size=12001) 2024-11-08T02:23:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-08T02:23:40,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032680558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032680558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032680558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032680558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032680570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032680768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032680768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032680769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032680769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:40,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032680778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-08T02:23:40,944 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/f4c4eeee599143a1991ef9936b4d1768 2024-11-08T02:23:40,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/a2f14dbfff1c4b1ea1d5b4e606b6e0d8 is 50, key is test_row_0/C:col10/1731032619772/Put/seqid=0 2024-11-08T02:23:40,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742081_1257 (size=12001) 2024-11-08T02:23:40,969 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/a2f14dbfff1c4b1ea1d5b4e606b6e0d8 2024-11-08T02:23:40,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/b36fc6ad594b4a98adf65e2fb366b27d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b36fc6ad594b4a98adf65e2fb366b27d 2024-11-08T02:23:40,977 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b36fc6ad594b4a98adf65e2fb366b27d, entries=150, sequenceid=88, filesize=11.7 K 2024-11-08T02:23:40,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/f4c4eeee599143a1991ef9936b4d1768 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/f4c4eeee599143a1991ef9936b4d1768 2024-11-08T02:23:40,982 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/f4c4eeee599143a1991ef9936b4d1768, entries=150, sequenceid=88, filesize=11.7 K 2024-11-08T02:23:40,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/a2f14dbfff1c4b1ea1d5b4e606b6e0d8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a2f14dbfff1c4b1ea1d5b4e606b6e0d8 2024-11-08T02:23:40,988 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a2f14dbfff1c4b1ea1d5b4e606b6e0d8, entries=150, sequenceid=88, filesize=11.7 K 2024-11-08T02:23:40,988 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1dba34e1c528fab2e73d505d0e433dae in 600ms, sequenceid=88, compaction requested=false 2024-11-08T02:23:40,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:40,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-08T02:23:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-08T02:23:40,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-08T02:23:40,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 755 msec 2024-11-08T02:23:40,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 761 msec 2024-11-08T02:23:41,037 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:23:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:41,082 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:41,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-08T02:23:41,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c39266773c404dc99202cf56f1d9aa58 is 50, key is test_row_0/A:col10/1731032621080/Put/seqid=0 2024-11-08T02:23:41,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032681088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032681089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032681089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032681090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032681090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742082_1258 (size=14341) 2024-11-08T02:23:41,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c39266773c404dc99202cf56f1d9aa58 2024-11-08T02:23:41,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/4f11b23f7cf2456db971206320a5898c is 50, key is test_row_0/B:col10/1731032621080/Put/seqid=0 2024-11-08T02:23:41,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742083_1259 (size=12001) 2024-11-08T02:23:41,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/4f11b23f7cf2456db971206320a5898c 2024-11-08T02:23:41,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/79f04cb81fd349b2afc204e92af5e751 is 50, key is test_row_0/C:col10/1731032621080/Put/seqid=0 2024-11-08T02:23:41,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032681198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032681198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032681198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032681198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742084_1260 (size=12001) 2024-11-08T02:23:41,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/79f04cb81fd349b2afc204e92af5e751 2024-11-08T02:23:41,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c39266773c404dc99202cf56f1d9aa58 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c39266773c404dc99202cf56f1d9aa58 2024-11-08T02:23:41,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c39266773c404dc99202cf56f1d9aa58, entries=200, sequenceid=120, filesize=14.0 K 2024-11-08T02:23:41,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/4f11b23f7cf2456db971206320a5898c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/4f11b23f7cf2456db971206320a5898c 2024-11-08T02:23:41,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/4f11b23f7cf2456db971206320a5898c, entries=150, sequenceid=120, filesize=11.7 K 2024-11-08T02:23:41,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/79f04cb81fd349b2afc204e92af5e751 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/79f04cb81fd349b2afc204e92af5e751 2024-11-08T02:23:41,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/79f04cb81fd349b2afc204e92af5e751, entries=150, sequenceid=120, filesize=11.7 K 2024-11-08T02:23:41,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for 1dba34e1c528fab2e73d505d0e433dae in 155ms, sequenceid=120, compaction requested=true 2024-11-08T02:23:41,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:41,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:41,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:41,238 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:41,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:41,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:41,238 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:41,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:41,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:41,239 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:41,239 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/A is initiating minor compaction (all files) 2024-11-08T02:23:41,239 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/A in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:41,239 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/9977d5e9ee4f44efaf3e8973bb2433cf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b36fc6ad594b4a98adf65e2fb366b27d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c39266773c404dc99202cf56f1d9aa58] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=37.6 K 2024-11-08T02:23:41,239 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9977d5e9ee4f44efaf3e8973bb2433cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731032619126 2024-11-08T02:23:41,240 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b36fc6ad594b4a98adf65e2fb366b27d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731032619770 2024-11-08T02:23:41,240 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:41,240 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/B is initiating minor compaction (all files) 2024-11-08T02:23:41,240 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/B in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:41,240 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/16fd880921264c12b00244cb755f0f34, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/f4c4eeee599143a1991ef9936b4d1768, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/4f11b23f7cf2456db971206320a5898c] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=35.3 K 2024-11-08T02:23:41,241 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c39266773c404dc99202cf56f1d9aa58, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731032620442 2024-11-08T02:23:41,241 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 16fd880921264c12b00244cb755f0f34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731032619126 2024-11-08T02:23:41,241 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting f4c4eeee599143a1991ef9936b4d1768, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731032619770 2024-11-08T02:23:41,242 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f11b23f7cf2456db971206320a5898c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731032621080 2024-11-08T02:23:41,259 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#A#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:41,259 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/fc230ea8a2714dd5b7dfb369269219a1 is 50, key is test_row_0/A:col10/1731032621080/Put/seqid=0 2024-11-08T02:23:41,268 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:41,269 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/854cef1a87c141aca4c94fd80f6ae176 is 50, key is test_row_0/B:col10/1731032621080/Put/seqid=0 2024-11-08T02:23:41,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742086_1262 (size=12241) 2024-11-08T02:23:41,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742085_1261 (size=12241) 2024-11-08T02:23:41,318 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/fc230ea8a2714dd5b7dfb369269219a1 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/fc230ea8a2714dd5b7dfb369269219a1 2024-11-08T02:23:41,327 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/A of 1dba34e1c528fab2e73d505d0e433dae into fc230ea8a2714dd5b7dfb369269219a1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:41,328 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:41,328 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/A, priority=13, startTime=1731032621237; duration=0sec 2024-11-08T02:23:41,328 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:41,328 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:A 2024-11-08T02:23:41,328 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:41,330 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:41,330 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/C is initiating minor compaction (all files) 2024-11-08T02:23:41,330 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/C in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:41,330 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/0ec3c52b2d4d48e6aaaa80ad290f9870, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a2f14dbfff1c4b1ea1d5b4e606b6e0d8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/79f04cb81fd349b2afc204e92af5e751] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=35.3 K 2024-11-08T02:23:41,330 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ec3c52b2d4d48e6aaaa80ad290f9870, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731032619126 2024-11-08T02:23:41,331 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f14dbfff1c4b1ea1d5b4e606b6e0d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731032619770 2024-11-08T02:23:41,331 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f04cb81fd349b2afc204e92af5e751, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731032621080 2024-11-08T02:23:41,339 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/854cef1a87c141aca4c94fd80f6ae176 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/854cef1a87c141aca4c94fd80f6ae176 2024-11-08T02:23:41,345 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/B of 1dba34e1c528fab2e73d505d0e433dae into 854cef1a87c141aca4c94fd80f6ae176(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:41,345 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:41,345 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/B, priority=13, startTime=1731032621238; duration=0sec 2024-11-08T02:23:41,345 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:41,345 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:B 2024-11-08T02:23:41,358 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#C#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:41,359 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/cfbf5b268340439d9bd9505bbb63df41 is 50, key is test_row_0/C:col10/1731032621080/Put/seqid=0 2024-11-08T02:23:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-08T02:23:41,369 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:41,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-08T02:23:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T02:23:41,409 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:41,410 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:41,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:41,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:23:41,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742087_1263 (size=12241) 2024-11-08T02:23:41,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c6ee39ad42e840119401c14328062871 is 50, key is test_row_0/A:col10/1731032621409/Put/seqid=0 2024-11-08T02:23:41,422 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/cfbf5b268340439d9bd9505bbb63df41 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/cfbf5b268340439d9bd9505bbb63df41 2024-11-08T02:23:41,427 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/C of 1dba34e1c528fab2e73d505d0e433dae into cfbf5b268340439d9bd9505bbb63df41(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:41,427 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:41,427 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/C, priority=13, startTime=1731032621238; duration=0sec 2024-11-08T02:23:41,427 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:41,427 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:C 2024-11-08T02:23:41,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742088_1264 (size=16881) 2024-11-08T02:23:41,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032681442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032681440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032681444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032681444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T02:23:41,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032681548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032681548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032681551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032681557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T02:23:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:41,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:41,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032681598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T02:23:41,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:41,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:41,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:41,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:41,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T02:23:41,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032681758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032681758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032681758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032681768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:41,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c6ee39ad42e840119401c14328062871 2024-11-08T02:23:41,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/e5fff8b66e8348ceaff5e768046f0606 is 50, key is test_row_0/B:col10/1731032621409/Put/seqid=0 2024-11-08T02:23:41,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T02:23:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:41,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:41,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:41,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742089_1265 (size=12151) 2024-11-08T02:23:41,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/e5fff8b66e8348ceaff5e768046f0606 2024-11-08T02:23:41,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/14675684861746769622821eb19b3ea9 is 50, key is test_row_0/C:col10/1731032621409/Put/seqid=0 2024-11-08T02:23:41,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742090_1266 (size=12151) 2024-11-08T02:23:42,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T02:23:42,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:42,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:42,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:42,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:42,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T02:23:42,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032682067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032682067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032682067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032682078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T02:23:42,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:42,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:42,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:42,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:42,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:42,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/14675684861746769622821eb19b3ea9 2024-11-08T02:23:42,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c6ee39ad42e840119401c14328062871 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c6ee39ad42e840119401c14328062871 2024-11-08T02:23:42,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c6ee39ad42e840119401c14328062871, entries=250, sequenceid=133, filesize=16.5 K 2024-11-08T02:23:42,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/e5fff8b66e8348ceaff5e768046f0606 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/e5fff8b66e8348ceaff5e768046f0606 2024-11-08T02:23:42,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/e5fff8b66e8348ceaff5e768046f0606, entries=150, sequenceid=133, filesize=11.9 K 2024-11-08T02:23:42,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/14675684861746769622821eb19b3ea9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/14675684861746769622821eb19b3ea9 2024-11-08T02:23:42,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/14675684861746769622821eb19b3ea9, entries=150, sequenceid=133, filesize=11.9 K 2024-11-08T02:23:42,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1dba34e1c528fab2e73d505d0e433dae in 919ms, sequenceid=133, compaction requested=false 2024-11-08T02:23:42,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:42,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T02:23:42,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:42,333 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:23:42,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/e3df008adf6a45a6a296cace9925a721 is 50, key is test_row_0/A:col10/1731032621438/Put/seqid=0 2024-11-08T02:23:42,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742091_1267 (size=12151) 2024-11-08T02:23:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T02:23:42,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:42,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:42,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032682587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032682590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032682591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032682594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032682618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,620 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 1dba34e1c528fab2e73d505d0e433dae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032682698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032682698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032682698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,747 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/e3df008adf6a45a6a296cace9925a721 2024-11-08T02:23:42,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/02eb53cf49b34c889aa1cd0f9b490b0f is 50, key is test_row_0/B:col10/1731032621438/Put/seqid=0 2024-11-08T02:23:42,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742092_1268 (size=12151) 2024-11-08T02:23:42,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032682908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032682907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032682908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:42,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:43,168 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/02eb53cf49b34c889aa1cd0f9b490b0f 2024-11-08T02:23:43,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/52f4f0aa5c684b4e8a6ef3836bc5500a is 50, key is test_row_0/C:col10/1731032621438/Put/seqid=0 2024-11-08T02:23:43,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742093_1269 (size=12151) 2024-11-08T02:23:43,208 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/52f4f0aa5c684b4e8a6ef3836bc5500a 2024-11-08T02:23:43,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/e3df008adf6a45a6a296cace9925a721 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e3df008adf6a45a6a296cace9925a721 2024-11-08T02:23:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032683218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032683217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:43,219 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e3df008adf6a45a6a296cace9925a721, entries=150, sequenceid=159, filesize=11.9 K 2024-11-08T02:23:43,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/02eb53cf49b34c889aa1cd0f9b490b0f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/02eb53cf49b34c889aa1cd0f9b490b0f 2024-11-08T02:23:43,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:43,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032683220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:43,226 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/02eb53cf49b34c889aa1cd0f9b490b0f, entries=150, sequenceid=159, filesize=11.9 K 2024-11-08T02:23:43,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/52f4f0aa5c684b4e8a6ef3836bc5500a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/52f4f0aa5c684b4e8a6ef3836bc5500a 2024-11-08T02:23:43,232 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/52f4f0aa5c684b4e8a6ef3836bc5500a, entries=150, sequenceid=159, filesize=11.9 K 2024-11-08T02:23:43,233 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1dba34e1c528fab2e73d505d0e433dae in 900ms, sequenceid=159, compaction requested=true 2024-11-08T02:23:43,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:43,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:43,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-08T02:23:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-08T02:23:43,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-08T02:23:43,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8240 sec 2024-11-08T02:23:43,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.8670 sec 2024-11-08T02:23:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T02:23:43,558 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:43,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-08T02:23:43,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T02:23:43,561 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:43,562 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:43,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:43,599 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:43,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:23:43,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/052ee0a0503e4921aae5113848b91526 is 50, key is test_row_0/A:col10/1731032622590/Put/seqid=0 2024-11-08T02:23:43,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742094_1270 (size=12151) 2024-11-08T02:23:43,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/052ee0a0503e4921aae5113848b91526 2024-11-08T02:23:43,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/5266b6b7e82e49f4b4c01ae5e096847b is 50, key is test_row_0/B:col10/1731032622590/Put/seqid=0 2024-11-08T02:23:43,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742095_1271 (size=12151) 2024-11-08T02:23:43,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T02:23:43,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:43,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032683693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:43,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:43,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T02:23:43,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:43,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:43,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:43,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:43,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:43,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032683728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:43,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032683728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:43,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032683728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:43,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032683808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:43,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T02:23:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:43,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:43,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T02:23:44,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032684017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:44,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T02:23:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:44,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:44,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:44,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:44,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/5266b6b7e82e49f4b4c01ae5e096847b 2024-11-08T02:23:44,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/80659acd29044bb392402a207f335523 is 50, key is test_row_0/C:col10/1731032622590/Put/seqid=0 2024-11-08T02:23:44,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742096_1272 (size=12151) 2024-11-08T02:23:44,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/80659acd29044bb392402a207f335523 2024-11-08T02:23:44,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/052ee0a0503e4921aae5113848b91526 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/052ee0a0503e4921aae5113848b91526 2024-11-08T02:23:44,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/052ee0a0503e4921aae5113848b91526, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T02:23:44,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/5266b6b7e82e49f4b4c01ae5e096847b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5266b6b7e82e49f4b4c01ae5e096847b 2024-11-08T02:23:44,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5266b6b7e82e49f4b4c01ae5e096847b, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T02:23:44,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/80659acd29044bb392402a207f335523 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/80659acd29044bb392402a207f335523 2024-11-08T02:23:44,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/80659acd29044bb392402a207f335523, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T02:23:44,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1dba34e1c528fab2e73d505d0e433dae in 486ms, sequenceid=171, compaction requested=true 2024-11-08T02:23:44,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:44,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:44,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:44,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:44,086 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:23:44,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:44,086 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:23:44,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:44,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:44,087 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:23:44,087 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/B is initiating minor compaction (all files) 2024-11-08T02:23:44,087 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/B in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:44,088 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/854cef1a87c141aca4c94fd80f6ae176, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/e5fff8b66e8348ceaff5e768046f0606, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/02eb53cf49b34c889aa1cd0f9b490b0f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5266b6b7e82e49f4b4c01ae5e096847b] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=47.6 K 2024-11-08T02:23:44,088 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53424 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:23:44,088 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/A is initiating minor compaction (all files) 2024-11-08T02:23:44,088 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/A in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:44,088 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/fc230ea8a2714dd5b7dfb369269219a1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c6ee39ad42e840119401c14328062871, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e3df008adf6a45a6a296cace9925a721, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/052ee0a0503e4921aae5113848b91526] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=52.2 K 2024-11-08T02:23:44,089 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 854cef1a87c141aca4c94fd80f6ae176, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731032621080 2024-11-08T02:23:44,089 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc230ea8a2714dd5b7dfb369269219a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731032621080 2024-11-08T02:23:44,089 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting e5fff8b66e8348ceaff5e768046f0606, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731032621409 2024-11-08T02:23:44,089 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6ee39ad42e840119401c14328062871, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731032621088 2024-11-08T02:23:44,089 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 02eb53cf49b34c889aa1cd0f9b490b0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731032621438 2024-11-08T02:23:44,089 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3df008adf6a45a6a296cace9925a721, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731032621438 2024-11-08T02:23:44,089 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 5266b6b7e82e49f4b4c01ae5e096847b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731032622588 2024-11-08T02:23:44,090 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 052ee0a0503e4921aae5113848b91526, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731032622588 2024-11-08T02:23:44,103 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#A#compaction#231 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:44,103 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/a7847bea20794bcca59250b066805e13 is 50, key is test_row_0/A:col10/1731032622590/Put/seqid=0 2024-11-08T02:23:44,107 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:44,107 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/6cddea1685a74ae0b18cdcd67efee628 is 50, key is test_row_0/B:col10/1731032622590/Put/seqid=0 2024-11-08T02:23:44,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742098_1274 (size=12527) 2024-11-08T02:23:44,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742097_1273 (size=12527) 2024-11-08T02:23:44,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T02:23:44,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:44,173 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:23:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T02:23:44,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/8571b5f8d74e44d899a86600bbbd03cd is 50, key is test_row_0/A:col10/1731032623692/Put/seqid=0 2024-11-08T02:23:44,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742099_1275 (size=12151) 2024-11-08T02:23:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:44,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:44,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032684366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:44,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032684478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:44,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:44,534 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/a7847bea20794bcca59250b066805e13 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/a7847bea20794bcca59250b066805e13 2024-11-08T02:23:44,538 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/6cddea1685a74ae0b18cdcd67efee628 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/6cddea1685a74ae0b18cdcd67efee628 2024-11-08T02:23:44,542 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/A of 1dba34e1c528fab2e73d505d0e433dae into a7847bea20794bcca59250b066805e13(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:44,542 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:44,542 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/A, priority=12, startTime=1731032624085; duration=0sec 2024-11-08T02:23:44,542 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:44,542 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:A 2024-11-08T02:23:44,542 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:23:44,545 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:23:44,545 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/C is initiating minor compaction (all files) 2024-11-08T02:23:44,545 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/C in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:44,545 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/cfbf5b268340439d9bd9505bbb63df41, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/14675684861746769622821eb19b3ea9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/52f4f0aa5c684b4e8a6ef3836bc5500a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/80659acd29044bb392402a207f335523] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=47.6 K 2024-11-08T02:23:44,545 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/B of 1dba34e1c528fab2e73d505d0e433dae into 6cddea1685a74ae0b18cdcd67efee628(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:44,546 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:44,546 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/B, priority=12, startTime=1731032624086; duration=0sec 2024-11-08T02:23:44,546 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfbf5b268340439d9bd9505bbb63df41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731032621080 2024-11-08T02:23:44,546 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:44,546 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:B 2024-11-08T02:23:44,546 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14675684861746769622821eb19b3ea9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731032621409 2024-11-08T02:23:44,546 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52f4f0aa5c684b4e8a6ef3836bc5500a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731032621438 2024-11-08T02:23:44,546 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80659acd29044bb392402a207f335523, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731032622588 2024-11-08T02:23:44,556 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:44,556 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/dc229689424549b2ac7019aabac28ba8 is 50, key is test_row_0/C:col10/1731032622590/Put/seqid=0 2024-11-08T02:23:44,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742100_1276 (size=12527) 2024-11-08T02:23:44,603 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/8571b5f8d74e44d899a86600bbbd03cd 2024-11-08T02:23:44,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/df5341bcd9094fe1b514c3bfd05b8515 is 50, key is test_row_0/B:col10/1731032623692/Put/seqid=0 2024-11-08T02:23:44,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742101_1277 (size=12151) 2024-11-08T02:23:44,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032684628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:44,632 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 1dba34e1c528fab2e73d505d0e433dae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:44,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032684688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T02:23:44,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032684737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:44,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032684737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:44,740 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 1dba34e1c528fab2e73d505d0e433dae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:44,740 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 1dba34e1c528fab2e73d505d0e433dae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2146 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:44,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:44,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032684748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:44,748 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 1dba34e1c528fab2e73d505d0e433dae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:44,967 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/dc229689424549b2ac7019aabac28ba8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/dc229689424549b2ac7019aabac28ba8 2024-11-08T02:23:44,972 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/C of 1dba34e1c528fab2e73d505d0e433dae into dc229689424549b2ac7019aabac28ba8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:44,972 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:44,972 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/C, priority=12, startTime=1731032624086; duration=0sec 2024-11-08T02:23:44,973 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:44,973 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:C 2024-11-08T02:23:44,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:44,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032684997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:44,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:44,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:45,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:45,026 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/df5341bcd9094fe1b514c3bfd05b8515 2024-11-08T02:23:45,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/76e6514ca0d44e52a334af2e7132b563 is 50, key is test_row_0/C:col10/1731032623692/Put/seqid=0 2024-11-08T02:23:45,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742102_1278 (size=12151) 2024-11-08T02:23:45,064 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/76e6514ca0d44e52a334af2e7132b563 2024-11-08T02:23:45,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/8571b5f8d74e44d899a86600bbbd03cd as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/8571b5f8d74e44d899a86600bbbd03cd 2024-11-08T02:23:45,078 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/8571b5f8d74e44d899a86600bbbd03cd, entries=150, sequenceid=195, filesize=11.9 K 2024-11-08T02:23:45,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/df5341bcd9094fe1b514c3bfd05b8515 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/df5341bcd9094fe1b514c3bfd05b8515 2024-11-08T02:23:45,082 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/df5341bcd9094fe1b514c3bfd05b8515, entries=150, sequenceid=195, filesize=11.9 K 2024-11-08T02:23:45,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/76e6514ca0d44e52a334af2e7132b563 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/76e6514ca0d44e52a334af2e7132b563 2024-11-08T02:23:45,087 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/76e6514ca0d44e52a334af2e7132b563, entries=150, sequenceid=195, filesize=11.9 K 2024-11-08T02:23:45,088 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1dba34e1c528fab2e73d505d0e433dae in 915ms, sequenceid=195, compaction requested=false 2024-11-08T02:23:45,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:45,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:45,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-08T02:23:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-08T02:23:45,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-08T02:23:45,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5280 sec 2024-11-08T02:23:45,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5340 sec 2024-11-08T02:23:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:45,509 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:45,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T02:23:45,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/802146d4b33846229451b2ec8e2ea39b is 50, key is test_row_0/A:col10/1731032624351/Put/seqid=0 2024-11-08T02:23:45,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742103_1279 (size=14541) 2024-11-08T02:23:45,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032685615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:45,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:45,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:45,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T02:23:45,708 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:45,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-08T02:23:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T02:23:45,711 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:45,712 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:45,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:45,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032685727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:45,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T02:23:45,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T02:23:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:45,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:45,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/802146d4b33846229451b2ec8e2ea39b 2024-11-08T02:23:45,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032685938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:45,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/573bb5081959448eba3b2cb1b6e0508a is 50, key is test_row_0/B:col10/1731032624351/Put/seqid=0 2024-11-08T02:23:45,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742104_1280 (size=12151) 2024-11-08T02:23:46,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T02:23:46,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:46,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:46,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:46,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:46,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T02:23:46,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T02:23:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:46,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:46,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:46,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:46,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:46,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032686247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:46,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T02:23:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:46,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:46,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:46,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:46,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T02:23:46,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/573bb5081959448eba3b2cb1b6e0508a 2024-11-08T02:23:46,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/756fd5915843455f9731a2a9fed3b490 is 50, key is test_row_0/C:col10/1731032624351/Put/seqid=0 2024-11-08T02:23:46,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742105_1281 (size=12151) 2024-11-08T02:23:46,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/756fd5915843455f9731a2a9fed3b490 2024-11-08T02:23:46,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/802146d4b33846229451b2ec8e2ea39b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/802146d4b33846229451b2ec8e2ea39b 2024-11-08T02:23:46,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/802146d4b33846229451b2ec8e2ea39b, entries=200, sequenceid=211, filesize=14.2 K 2024-11-08T02:23:46,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/573bb5081959448eba3b2cb1b6e0508a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/573bb5081959448eba3b2cb1b6e0508a 2024-11-08T02:23:46,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/573bb5081959448eba3b2cb1b6e0508a, entries=150, sequenceid=211, filesize=11.9 K 2024-11-08T02:23:46,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/756fd5915843455f9731a2a9fed3b490 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/756fd5915843455f9731a2a9fed3b490 2024-11-08T02:23:46,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/756fd5915843455f9731a2a9fed3b490, entries=150, sequenceid=211, filesize=11.9 K 2024-11-08T02:23:46,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1dba34e1c528fab2e73d505d0e433dae in 934ms, sequenceid=211, compaction requested=true 2024-11-08T02:23:46,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:46,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:46,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:46,443 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:46,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:46,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:46,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:46,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T02:23:46,443 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:46,445 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:46,445 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/B is initiating minor compaction (all files) 2024-11-08T02:23:46,445 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/B in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:46,445 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/6cddea1685a74ae0b18cdcd67efee628, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/df5341bcd9094fe1b514c3bfd05b8515, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/573bb5081959448eba3b2cb1b6e0508a] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=36.0 K 2024-11-08T02:23:46,445 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:46,445 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/A is initiating minor compaction (all files) 2024-11-08T02:23:46,445 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/A in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:46,445 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/a7847bea20794bcca59250b066805e13, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/8571b5f8d74e44d899a86600bbbd03cd, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/802146d4b33846229451b2ec8e2ea39b] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=38.3 K 2024-11-08T02:23:46,446 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cddea1685a74ae0b18cdcd67efee628, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731032622588 2024-11-08T02:23:46,446 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting df5341bcd9094fe1b514c3bfd05b8515, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032623681 2024-11-08T02:23:46,447 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a7847bea20794bcca59250b066805e13, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731032622588 2024-11-08T02:23:46,448 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 573bb5081959448eba3b2cb1b6e0508a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731032624339 2024-11-08T02:23:46,448 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 8571b5f8d74e44d899a86600bbbd03cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032623681 2024-11-08T02:23:46,448 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 802146d4b33846229451b2ec8e2ea39b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731032624339 2024-11-08T02:23:46,466 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:46,467 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/6f1c64870022424db04ffd6a71f947ce is 50, key is test_row_0/A:col10/1731032624351/Put/seqid=0 2024-11-08T02:23:46,469 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:46,470 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/c5dac2d1d9c04b9791b90287b405a54e is 50, key is test_row_0/B:col10/1731032624351/Put/seqid=0 2024-11-08T02:23:46,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T02:23:46,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:46,479 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T02:23:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/0180e8b3d7af4cb2b57e8522cfef5541 is 50, key is test_row_0/A:col10/1731032625602/Put/seqid=0 2024-11-08T02:23:46,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742107_1283 (size=12629) 2024-11-08T02:23:46,515 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/c5dac2d1d9c04b9791b90287b405a54e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/c5dac2d1d9c04b9791b90287b405a54e 2024-11-08T02:23:46,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742106_1282 (size=12629) 2024-11-08T02:23:46,526 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/B of 1dba34e1c528fab2e73d505d0e433dae into c5dac2d1d9c04b9791b90287b405a54e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:46,526 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:46,526 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/B, priority=13, startTime=1731032626443; duration=0sec 2024-11-08T02:23:46,526 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:46,526 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:B 2024-11-08T02:23:46,526 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:46,527 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:46,528 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/C is initiating minor compaction (all files) 2024-11-08T02:23:46,528 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/C in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:46,528 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/dc229689424549b2ac7019aabac28ba8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/76e6514ca0d44e52a334af2e7132b563, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/756fd5915843455f9731a2a9fed3b490] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=36.0 K 2024-11-08T02:23:46,528 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc229689424549b2ac7019aabac28ba8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731032622588 2024-11-08T02:23:46,529 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/6f1c64870022424db04ffd6a71f947ce as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/6f1c64870022424db04ffd6a71f947ce 2024-11-08T02:23:46,530 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76e6514ca0d44e52a334af2e7132b563, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032623681 2024-11-08T02:23:46,530 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 756fd5915843455f9731a2a9fed3b490, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731032624339 2024-11-08T02:23:46,535 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/A of 1dba34e1c528fab2e73d505d0e433dae into 6f1c64870022424db04ffd6a71f947ce(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:46,535 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:46,535 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/A, priority=13, startTime=1731032626443; duration=0sec 2024-11-08T02:23:46,535 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:46,536 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:A 2024-11-08T02:23:46,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742108_1284 (size=12151) 2024-11-08T02:23:46,567 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:46,568 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/6ce71e007dc2439abfba83d952a0dec1 is 50, key is test_row_0/C:col10/1731032624351/Put/seqid=0 2024-11-08T02:23:46,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742109_1285 (size=12629) 2024-11-08T02:23:46,608 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/6ce71e007dc2439abfba83d952a0dec1 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/6ce71e007dc2439abfba83d952a0dec1 2024-11-08T02:23:46,615 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/C of 1dba34e1c528fab2e73d505d0e433dae into 6ce71e007dc2439abfba83d952a0dec1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:46,615 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:46,615 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/C, priority=13, startTime=1731032626443; duration=0sec 2024-11-08T02:23:46,615 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:46,615 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:C 2024-11-08T02:23:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:46,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:46,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032686765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:46,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032686766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:46,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032686768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:46,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032686768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:46,770 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 1dba34e1c528fab2e73d505d0e433dae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:46,770 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 1dba34e1c528fab2e73d505d0e433dae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T02:23:46,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032686878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:46,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032686878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:46,946 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/0180e8b3d7af4cb2b57e8522cfef5541 2024-11-08T02:23:46,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/83ff99db190a42a5a80fef82bba2f4f7 is 50, key is test_row_0/B:col10/1731032625602/Put/seqid=0 2024-11-08T02:23:46,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742110_1286 (size=12151) 2024-11-08T02:23:46,962 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/83ff99db190a42a5a80fef82bba2f4f7 2024-11-08T02:23:46,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/c757dc3bd7814db79945c65fc2305b94 is 50, key is test_row_0/C:col10/1731032625602/Put/seqid=0 2024-11-08T02:23:47,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742111_1287 (size=12151) 2024-11-08T02:23:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032687088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032687088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:47,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032687398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032687398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:47,402 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/c757dc3bd7814db79945c65fc2305b94 2024-11-08T02:23:47,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/0180e8b3d7af4cb2b57e8522cfef5541 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0180e8b3d7af4cb2b57e8522cfef5541 2024-11-08T02:23:47,418 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0180e8b3d7af4cb2b57e8522cfef5541, entries=150, sequenceid=234, filesize=11.9 K 2024-11-08T02:23:47,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/83ff99db190a42a5a80fef82bba2f4f7 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/83ff99db190a42a5a80fef82bba2f4f7 2024-11-08T02:23:47,423 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/83ff99db190a42a5a80fef82bba2f4f7, entries=150, sequenceid=234, filesize=11.9 K 2024-11-08T02:23:47,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/c757dc3bd7814db79945c65fc2305b94 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/c757dc3bd7814db79945c65fc2305b94 2024-11-08T02:23:47,428 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/c757dc3bd7814db79945c65fc2305b94, entries=150, sequenceid=234, filesize=11.9 K 2024-11-08T02:23:47,429 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1dba34e1c528fab2e73d505d0e433dae in 950ms, sequenceid=234, compaction requested=false 2024-11-08T02:23:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-08T02:23:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-08T02:23:47,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-08T02:23:47,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-11-08T02:23:47,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7240 sec 2024-11-08T02:23:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T02:23:47,858 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:47,858 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-08T02:23:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T02:23:47,860 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:47,861 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:47,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:47,910 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:47,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T02:23:47,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/6a245d3d4356414c90613a8863091171 is 50, key is test_row_0/A:col10/1731032626762/Put/seqid=0 2024-11-08T02:23:47,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742112_1288 (size=14541) 2024-11-08T02:23:47,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/6a245d3d4356414c90613a8863091171 2024-11-08T02:23:47,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032687952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:47,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032687957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:47,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/3e25751b205f4a969a3dbe2a9f76eba7 is 50, key is test_row_0/B:col10/1731032626762/Put/seqid=0 2024-11-08T02:23:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T02:23:47,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742113_1289 (size=12151) 2024-11-08T02:23:47,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/3e25751b205f4a969a3dbe2a9f76eba7 2024-11-08T02:23:47,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/a5a8255abccd484f977cdc9b1936300b is 50, key is test_row_0/C:col10/1731032626762/Put/seqid=0 2024-11-08T02:23:47,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742114_1290 (size=12151) 2024-11-08T02:23:48,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T02:23:48,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:48,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:48,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:48,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:48,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:48,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032688058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:48,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032688068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:48,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:48,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T02:23:48,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:48,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:48,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:48,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:48,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T02:23:48,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032688268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:48,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032688281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:48,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:48,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T02:23:48,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:48,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:48,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:48,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:48,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:48,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/a5a8255abccd484f977cdc9b1936300b 2024-11-08T02:23:48,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/6a245d3d4356414c90613a8863091171 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/6a245d3d4356414c90613a8863091171 2024-11-08T02:23:48,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/6a245d3d4356414c90613a8863091171, entries=200, sequenceid=252, filesize=14.2 K 2024-11-08T02:23:48,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/3e25751b205f4a969a3dbe2a9f76eba7 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/3e25751b205f4a969a3dbe2a9f76eba7 2024-11-08T02:23:48,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/3e25751b205f4a969a3dbe2a9f76eba7, entries=150, sequenceid=252, filesize=11.9 K 2024-11-08T02:23:48,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/a5a8255abccd484f977cdc9b1936300b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a5a8255abccd484f977cdc9b1936300b 2024-11-08T02:23:48,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a5a8255abccd484f977cdc9b1936300b, entries=150, sequenceid=252, filesize=11.9 K 2024-11-08T02:23:48,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1dba34e1c528fab2e73d505d0e433dae in 508ms, sequenceid=252, compaction requested=true 2024-11-08T02:23:48,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:48,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:48,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:48,419 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:48,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:48,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:48,419 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:48,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:48,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:48,420 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:48,420 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:48,420 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/A is initiating minor compaction (all files) 2024-11-08T02:23:48,420 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/B is initiating minor compaction (all files) 2024-11-08T02:23:48,420 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/B in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:48,420 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/A in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:48,420 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/6f1c64870022424db04ffd6a71f947ce, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0180e8b3d7af4cb2b57e8522cfef5541, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/6a245d3d4356414c90613a8863091171] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=38.4 K 2024-11-08T02:23:48,420 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/c5dac2d1d9c04b9791b90287b405a54e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/83ff99db190a42a5a80fef82bba2f4f7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/3e25751b205f4a969a3dbe2a9f76eba7] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=36.1 K 2024-11-08T02:23:48,420 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c5dac2d1d9c04b9791b90287b405a54e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731032624339 2024-11-08T02:23:48,420 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f1c64870022424db04ffd6a71f947ce, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731032624339 2024-11-08T02:23:48,421 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 83ff99db190a42a5a80fef82bba2f4f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731032625602 2024-11-08T02:23:48,421 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0180e8b3d7af4cb2b57e8522cfef5541, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731032625602 2024-11-08T02:23:48,421 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e25751b205f4a969a3dbe2a9f76eba7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731032626758 2024-11-08T02:23:48,421 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a245d3d4356414c90613a8863091171, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731032626758 2024-11-08T02:23:48,445 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:48,445 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:48,446 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/9784ffee8edf4344884f0ed5efa1fc64 is 50, key is test_row_0/A:col10/1731032626762/Put/seqid=0 2024-11-08T02:23:48,446 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/535700f7129843659216975b3d6e2583 is 50, key is test_row_0/B:col10/1731032626762/Put/seqid=0 2024-11-08T02:23:48,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742115_1291 (size=12731) 2024-11-08T02:23:48,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742116_1292 (size=12731) 2024-11-08T02:23:48,475 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/535700f7129843659216975b3d6e2583 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/535700f7129843659216975b3d6e2583 2024-11-08T02:23:48,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T02:23:48,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:48,477 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T02:23:48,482 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/B of 1dba34e1c528fab2e73d505d0e433dae into 535700f7129843659216975b3d6e2583(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:48,482 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:48,482 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/B, priority=13, startTime=1731032628419; duration=0sec 2024-11-08T02:23:48,482 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:48,482 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:B 2024-11-08T02:23:48,482 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:23:48,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c1b3bce427ea42d2bd021a6b95ac7b55 is 50, key is test_row_0/A:col10/1731032627950/Put/seqid=0 2024-11-08T02:23:48,483 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:23:48,483 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/C is initiating minor compaction (all files) 2024-11-08T02:23:48,483 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/C in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:48,483 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/6ce71e007dc2439abfba83d952a0dec1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/c757dc3bd7814db79945c65fc2305b94, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a5a8255abccd484f977cdc9b1936300b] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=36.1 K 2024-11-08T02:23:48,484 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce71e007dc2439abfba83d952a0dec1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731032624339 2024-11-08T02:23:48,484 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c757dc3bd7814db79945c65fc2305b94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731032625602 2024-11-08T02:23:48,484 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a5a8255abccd484f977cdc9b1936300b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731032626758 2024-11-08T02:23:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T02:23:48,494 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:48,495 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/5db0aafca1054a5a8706c72f80ca5af1 is 50, key is test_row_0/C:col10/1731032626762/Put/seqid=0 2024-11-08T02:23:48,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742117_1293 (size=12301) 2024-11-08T02:23:48,508 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c1b3bce427ea42d2bd021a6b95ac7b55 2024-11-08T02:23:48,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742118_1294 (size=12731) 2024-11-08T02:23:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/05f3affd922d4e8185186acabe652cb4 is 50, key is test_row_0/B:col10/1731032627950/Put/seqid=0 2024-11-08T02:23:48,523 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/5db0aafca1054a5a8706c72f80ca5af1 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/5db0aafca1054a5a8706c72f80ca5af1 2024-11-08T02:23:48,528 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/C of 1dba34e1c528fab2e73d505d0e433dae into 5db0aafca1054a5a8706c72f80ca5af1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:48,528 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:48,528 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/C, priority=13, startTime=1731032628419; duration=0sec 2024-11-08T02:23:48,528 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:48,528 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:C 2024-11-08T02:23:48,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742119_1295 (size=12301) 2024-11-08T02:23:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:48,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:48,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032688607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:48,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032688610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:48,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:48,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39310 deadline: 1731032688657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:48,660 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 1dba34e1c528fab2e73d505d0e433dae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:48,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032688718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:48,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032688718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:48,875 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/9784ffee8edf4344884f0ed5efa1fc64 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/9784ffee8edf4344884f0ed5efa1fc64 2024-11-08T02:23:48,881 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/A of 1dba34e1c528fab2e73d505d0e433dae into 9784ffee8edf4344884f0ed5efa1fc64(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:48,881 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:48,881 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/A, priority=13, startTime=1731032628419; duration=0sec 2024-11-08T02:23:48,881 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:48,881 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:A 2024-11-08T02:23:48,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032688928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:48,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032688928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:48,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:48,946 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/05f3affd922d4e8185186acabe652cb4 2024-11-08T02:23:48,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/a56505f79f6449f3be729ac2be6e066a is 50, key is test_row_0/C:col10/1731032627950/Put/seqid=0 2024-11-08T02:23:48,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742120_1296 (size=12301) 2024-11-08T02:23:48,959 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/a56505f79f6449f3be729ac2be6e066a 2024-11-08T02:23:48,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c1b3bce427ea42d2bd021a6b95ac7b55 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c1b3bce427ea42d2bd021a6b95ac7b55 2024-11-08T02:23:48,968 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c1b3bce427ea42d2bd021a6b95ac7b55, entries=150, sequenceid=273, filesize=12.0 K 2024-11-08T02:23:48,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/05f3affd922d4e8185186acabe652cb4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/05f3affd922d4e8185186acabe652cb4 2024-11-08T02:23:48,975 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/05f3affd922d4e8185186acabe652cb4, entries=150, sequenceid=273, filesize=12.0 K 2024-11-08T02:23:48,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/a56505f79f6449f3be729ac2be6e066a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a56505f79f6449f3be729ac2be6e066a 2024-11-08T02:23:48,981 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a56505f79f6449f3be729ac2be6e066a, entries=150, sequenceid=273, filesize=12.0 K 2024-11-08T02:23:48,982 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 1dba34e1c528fab2e73d505d0e433dae in 505ms, sequenceid=273, compaction requested=false 2024-11-08T02:23:48,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:48,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:48,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-08T02:23:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-08T02:23:48,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-08T02:23:48,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1220 sec 2024-11-08T02:23:48,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1270 sec 2024-11-08T02:23:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T02:23:48,999 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:48,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-08T02:23:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T02:23:49,001 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:49,002 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:49,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T02:23:49,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T02:23:49,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:49,154 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T02:23:49,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c3a51134c43a4c978d02a8661028fbf2 is 50, key is test_row_0/A:col10/1731032628609/Put/seqid=0 2024-11-08T02:23:49,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742121_1297 (size=12301) 2024-11-08T02:23:49,164 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c3a51134c43a4c978d02a8661028fbf2 2024-11-08T02:23:49,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/b908207c396446ff8e9ef6815cd48a0e is 50, key is test_row_0/B:col10/1731032628609/Put/seqid=0 2024-11-08T02:23:49,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742122_1298 (size=12301) 2024-11-08T02:23:49,178 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/b908207c396446ff8e9ef6815cd48a0e 2024-11-08T02:23:49,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/8d00a73473c2419292332b669f7f05e7 is 50, key is test_row_0/C:col10/1731032628609/Put/seqid=0 2024-11-08T02:23:49,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742123_1299 (size=12301) 2024-11-08T02:23:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:49,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032689279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:49,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:49,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032689280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T02:23:49,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032689387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:49,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032689388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:49,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:49,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032689598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032689598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:49,603 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/8d00a73473c2419292332b669f7f05e7 2024-11-08T02:23:49,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/c3a51134c43a4c978d02a8661028fbf2 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c3a51134c43a4c978d02a8661028fbf2 2024-11-08T02:23:49,614 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c3a51134c43a4c978d02a8661028fbf2, entries=150, sequenceid=291, filesize=12.0 K 2024-11-08T02:23:49,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/b908207c396446ff8e9ef6815cd48a0e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b908207c396446ff8e9ef6815cd48a0e 2024-11-08T02:23:49,620 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b908207c396446ff8e9ef6815cd48a0e, entries=150, sequenceid=291, filesize=12.0 K 2024-11-08T02:23:49,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/8d00a73473c2419292332b669f7f05e7 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/8d00a73473c2419292332b669f7f05e7 2024-11-08T02:23:49,625 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/8d00a73473c2419292332b669f7f05e7, entries=150, sequenceid=291, filesize=12.0 K 2024-11-08T02:23:49,626 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 1dba34e1c528fab2e73d505d0e433dae in 472ms, sequenceid=291, compaction requested=true 2024-11-08T02:23:49,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:49,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:49,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-08T02:23:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-08T02:23:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T02:23:49,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-08T02:23:49,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 625 msec 2024-11-08T02:23:49,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 632 msec 2024-11-08T02:23:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:49,911 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:49,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T02:23:49,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/b5a3236f1ac04a2e90c77bf2446230af is 50, key is test_row_0/A:col10/1731032629279/Put/seqid=0 2024-11-08T02:23:49,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742124_1300 (size=14741) 2024-11-08T02:23:49,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032689933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:49,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032689936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:50,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032690048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:50,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032690048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T02:23:50,138 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:50,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-08T02:23:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T02:23:50,140 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:50,141 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:50,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:50,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T02:23:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032690257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032690258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:50,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T02:23:50,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:50,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:50,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:50,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/b5a3236f1ac04a2e90c77bf2446230af 2024-11-08T02:23:50,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/b56dab8bcea2411ca148d5099a79386a is 50, key is test_row_0/B:col10/1731032629279/Put/seqid=0 2024-11-08T02:23:50,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742125_1301 (size=12301) 2024-11-08T02:23:50,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T02:23:50,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:50,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:50,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:50,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T02:23:50,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032690568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:50,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032690568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:50,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T02:23:50,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:50,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:50,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:50,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T02:23:50,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:50,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:50,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:50,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/b56dab8bcea2411ca148d5099a79386a 2024-11-08T02:23:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T02:23:50,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/e18447cbb8d64056921d761b2217b269 is 50, key is test_row_0/C:col10/1731032629279/Put/seqid=0 2024-11-08T02:23:50,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39280 deadline: 1731032690778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:50,779 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 1dba34e1c528fab2e73d505d0e433dae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:50,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742126_1302 (size=12301) 2024-11-08T02:23:50,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39252 deadline: 1731032690788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:50,791 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 1dba34e1c528fab2e73d505d0e433dae of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:23:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:50,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:50,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T02:23:50,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:50,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:50,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:50,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:51,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T02:23:51,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:51,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:51,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:51,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:51,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:51,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032691078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:51,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032691078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:51,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/e18447cbb8d64056921d761b2217b269 2024-11-08T02:23:51,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/b5a3236f1ac04a2e90c77bf2446230af as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b5a3236f1ac04a2e90c77bf2446230af 2024-11-08T02:23:51,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b5a3236f1ac04a2e90c77bf2446230af, entries=200, sequenceid=314, filesize=14.4 K 2024-11-08T02:23:51,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/b56dab8bcea2411ca148d5099a79386a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b56dab8bcea2411ca148d5099a79386a 2024-11-08T02:23:51,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T02:23:51,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:51,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:51,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:51,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:51,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:51,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b56dab8bcea2411ca148d5099a79386a, entries=150, sequenceid=314, filesize=12.0 K 2024-11-08T02:23:51,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/e18447cbb8d64056921d761b2217b269 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/e18447cbb8d64056921d761b2217b269 2024-11-08T02:23:51,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/e18447cbb8d64056921d761b2217b269, entries=150, sequenceid=314, filesize=12.0 K 2024-11-08T02:23:51,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1dba34e1c528fab2e73d505d0e433dae in 1311ms, sequenceid=314, compaction requested=true 2024-11-08T02:23:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:51,222 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:23:51,222 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:23:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:51,223 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:23:51,223 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52074 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:23:51,223 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/B is initiating minor compaction (all files) 2024-11-08T02:23:51,223 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/A is initiating minor compaction (all files) 2024-11-08T02:23:51,223 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/B in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:51,223 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/A in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:51,223 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/9784ffee8edf4344884f0ed5efa1fc64, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c1b3bce427ea42d2bd021a6b95ac7b55, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c3a51134c43a4c978d02a8661028fbf2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b5a3236f1ac04a2e90c77bf2446230af] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=50.9 K 2024-11-08T02:23:51,223 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/535700f7129843659216975b3d6e2583, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/05f3affd922d4e8185186acabe652cb4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b908207c396446ff8e9ef6815cd48a0e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b56dab8bcea2411ca148d5099a79386a] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=48.5 K 2024-11-08T02:23:51,224 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 535700f7129843659216975b3d6e2583, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731032626758 2024-11-08T02:23:51,224 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9784ffee8edf4344884f0ed5efa1fc64, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731032626758 2024-11-08T02:23:51,224 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1b3bce427ea42d2bd021a6b95ac7b55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731032627938 2024-11-08T02:23:51,224 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 05f3affd922d4e8185186acabe652cb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731032627938 2024-11-08T02:23:51,224 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a51134c43a4c978d02a8661028fbf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731032628600 2024-11-08T02:23:51,224 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting b908207c396446ff8e9ef6815cd48a0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731032628600 2024-11-08T02:23:51,224 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5a3236f1ac04a2e90c77bf2446230af, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731032629278 2024-11-08T02:23:51,224 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting b56dab8bcea2411ca148d5099a79386a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731032629279 2024-11-08T02:23:51,235 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#A#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:51,235 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#B#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:51,235 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/005380dd31e7497ab1f8caf2ea442ef4 is 50, key is test_row_0/A:col10/1731032629279/Put/seqid=0 2024-11-08T02:23:51,235 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/7fcf946d9b4f49b89009b5bccf35b71a is 50, key is test_row_0/B:col10/1731032629279/Put/seqid=0 2024-11-08T02:23:51,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742128_1304 (size=13017) 2024-11-08T02:23:51,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742127_1303 (size=13017) 2024-11-08T02:23:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T02:23:51,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T02:23:51,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:51,364 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T02:23:51,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/f40137d79ff54399a3bdf3ae93339bcf is 50, key is test_row_0/A:col10/1731032629934/Put/seqid=0 2024-11-08T02:23:51,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742129_1305 (size=12301) 2024-11-08T02:23:51,677 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/005380dd31e7497ab1f8caf2ea442ef4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/005380dd31e7497ab1f8caf2ea442ef4 2024-11-08T02:23:51,683 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/7fcf946d9b4f49b89009b5bccf35b71a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/7fcf946d9b4f49b89009b5bccf35b71a 2024-11-08T02:23:51,685 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/A of 1dba34e1c528fab2e73d505d0e433dae into 005380dd31e7497ab1f8caf2ea442ef4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:51,685 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:51,685 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/A, priority=12, startTime=1731032631222; duration=0sec 2024-11-08T02:23:51,685 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:51,685 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:A 2024-11-08T02:23:51,685 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:23:51,686 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:23:51,686 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/C is initiating minor compaction (all files) 2024-11-08T02:23:51,686 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/C in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:51,686 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/5db0aafca1054a5a8706c72f80ca5af1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a56505f79f6449f3be729ac2be6e066a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/8d00a73473c2419292332b669f7f05e7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/e18447cbb8d64056921d761b2217b269] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=48.5 K 2024-11-08T02:23:51,687 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5db0aafca1054a5a8706c72f80ca5af1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731032626758 2024-11-08T02:23:51,688 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56505f79f6449f3be729ac2be6e066a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731032627938 2024-11-08T02:23:51,688 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/B of 1dba34e1c528fab2e73d505d0e433dae into 7fcf946d9b4f49b89009b5bccf35b71a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:51,688 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:51,688 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/B, priority=12, startTime=1731032631222; duration=0sec 2024-11-08T02:23:51,688 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d00a73473c2419292332b669f7f05e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731032628600 2024-11-08T02:23:51,688 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:51,688 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:B 2024-11-08T02:23:51,688 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting e18447cbb8d64056921d761b2217b269, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731032629279 2024-11-08T02:23:51,697 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#C#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:51,698 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/3d95cdec8abe451d81d426d639aea9a9 is 50, key is test_row_0/C:col10/1731032629279/Put/seqid=0 2024-11-08T02:23:51,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742130_1306 (size=13017) 2024-11-08T02:23:51,708 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/3d95cdec8abe451d81d426d639aea9a9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/3d95cdec8abe451d81d426d639aea9a9 2024-11-08T02:23:51,724 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/C of 1dba34e1c528fab2e73d505d0e433dae into 3d95cdec8abe451d81d426d639aea9a9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:51,724 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:51,724 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/C, priority=12, startTime=1731032631222; duration=0sec 2024-11-08T02:23:51,724 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:51,724 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:C 2024-11-08T02:23:51,823 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/f40137d79ff54399a3bdf3ae93339bcf 2024-11-08T02:23:51,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/9d7eb8e77b4d4cc38acc327d4159aa89 is 50, key is test_row_0/B:col10/1731032629934/Put/seqid=0 2024-11-08T02:23:51,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742131_1307 (size=12301) 2024-11-08T02:23:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:52,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:52,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:52,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032692130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032692130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:52,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:52,239 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/9d7eb8e77b4d4cc38acc327d4159aa89 2024-11-08T02:23:52,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032692238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:52,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032692238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:52,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/d174a893d2694174846893fdf7311453 is 50, key is test_row_0/C:col10/1731032629934/Put/seqid=0 2024-11-08T02:23:52,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742132_1308 (size=12301) 2024-11-08T02:23:52,249 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/d174a893d2694174846893fdf7311453 2024-11-08T02:23:52,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/f40137d79ff54399a3bdf3ae93339bcf as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f40137d79ff54399a3bdf3ae93339bcf 2024-11-08T02:23:52,258 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f40137d79ff54399a3bdf3ae93339bcf, entries=150, sequenceid=327, filesize=12.0 K 2024-11-08T02:23:52,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/9d7eb8e77b4d4cc38acc327d4159aa89 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9d7eb8e77b4d4cc38acc327d4159aa89 2024-11-08T02:23:52,263 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9d7eb8e77b4d4cc38acc327d4159aa89, entries=150, sequenceid=327, filesize=12.0 K 2024-11-08T02:23:52,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/d174a893d2694174846893fdf7311453 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d174a893d2694174846893fdf7311453 2024-11-08T02:23:52,268 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d174a893d2694174846893fdf7311453, entries=150, sequenceid=327, filesize=12.0 K 2024-11-08T02:23:52,269 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 1dba34e1c528fab2e73d505d0e433dae in 904ms, sequenceid=327, compaction requested=false 2024-11-08T02:23:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:52,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-08T02:23:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-08T02:23:52,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-08T02:23:52,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1290 sec 2024-11-08T02:23:52,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.1340 sec 2024-11-08T02:23:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T02:23:52,288 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:52,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-08T02:23:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T02:23:52,290 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:52,291 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:52,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T02:23:52,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T02:23:52,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:52,443 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:23:52,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/0f569f59afae4eaaab6a2bc68c7273a4 is 50, key is test_row_0/A:col10/1731032632123/Put/seqid=0 2024-11-08T02:23:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:52,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:52,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742133_1309 (size=12301) 2024-11-08T02:23:52,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032692464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:52,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032692468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032692577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032692577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T02:23:52,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032692788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:52,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032692788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:52,854 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/0f569f59afae4eaaab6a2bc68c7273a4 2024-11-08T02:23:52,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/aa47c3bc6f8c4a6a8ea54cf33afad9b5 is 50, key is test_row_0/B:col10/1731032632123/Put/seqid=0 2024-11-08T02:23:52,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742134_1310 (size=12301) 2024-11-08T02:23:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T02:23:53,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032693098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:53,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032693098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:53,282 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/aa47c3bc6f8c4a6a8ea54cf33afad9b5 2024-11-08T02:23:53,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/37f5ad237b454e9687f86507d0b3612e is 50, key is test_row_0/C:col10/1731032632123/Put/seqid=0 2024-11-08T02:23:53,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742135_1311 (size=12301) 2024-11-08T02:23:53,305 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/37f5ad237b454e9687f86507d0b3612e 2024-11-08T02:23:53,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/0f569f59afae4eaaab6a2bc68c7273a4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0f569f59afae4eaaab6a2bc68c7273a4 2024-11-08T02:23:53,314 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0f569f59afae4eaaab6a2bc68c7273a4, entries=150, sequenceid=354, filesize=12.0 K 2024-11-08T02:23:53,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/aa47c3bc6f8c4a6a8ea54cf33afad9b5 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/aa47c3bc6f8c4a6a8ea54cf33afad9b5 2024-11-08T02:23:53,318 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/aa47c3bc6f8c4a6a8ea54cf33afad9b5, entries=150, sequenceid=354, filesize=12.0 K 2024-11-08T02:23:53,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/37f5ad237b454e9687f86507d0b3612e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/37f5ad237b454e9687f86507d0b3612e 2024-11-08T02:23:53,323 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/37f5ad237b454e9687f86507d0b3612e, entries=150, sequenceid=354, filesize=12.0 K 2024-11-08T02:23:53,324 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1dba34e1c528fab2e73d505d0e433dae in 881ms, sequenceid=354, compaction requested=true 2024-11-08T02:23:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-08T02:23:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-08T02:23:53,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-08T02:23:53,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0350 sec 2024-11-08T02:23:53,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0400 sec 2024-11-08T02:23:53,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T02:23:53,428 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:53,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-08T02:23:53,431 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T02:23:53,432 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:53,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T02:23:53,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T02:23:53,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:53,585 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:23:53,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/e4b605b283c94135a3b9955bcce2efe0 is 50, key is test_row_0/A:col10/1731032632462/Put/seqid=0 2024-11-08T02:23:53,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742136_1312 (size=12301) 2024-11-08T02:23:53,606 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/e4b605b283c94135a3b9955bcce2efe0 2024-11-08T02:23:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:53,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:53,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/9cf409db5380465180ce188af728308a is 50, key is test_row_0/B:col10/1731032632462/Put/seqid=0 2024-11-08T02:23:53,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742137_1313 (size=12301) 2024-11-08T02:23:53,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032693653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:53,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:53,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032693656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T02:23:53,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032693768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:53,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032693768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:53,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:53,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032693977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:53,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032693977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:54,030 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/9cf409db5380465180ce188af728308a 2024-11-08T02:23:54,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/3412f8705bc945388f5f43c941b22779 is 50, key is test_row_0/C:col10/1731032632462/Put/seqid=0 2024-11-08T02:23:54,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742138_1314 (size=12301) 2024-11-08T02:23:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T02:23:54,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032694288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:54,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032694288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:54,443 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/3412f8705bc945388f5f43c941b22779 2024-11-08T02:23:54,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/e4b605b283c94135a3b9955bcce2efe0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e4b605b283c94135a3b9955bcce2efe0 2024-11-08T02:23:54,457 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e4b605b283c94135a3b9955bcce2efe0, entries=150, sequenceid=366, filesize=12.0 K 2024-11-08T02:23:54,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/9cf409db5380465180ce188af728308a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9cf409db5380465180ce188af728308a 2024-11-08T02:23:54,464 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9cf409db5380465180ce188af728308a, entries=150, sequenceid=366, filesize=12.0 K 2024-11-08T02:23:54,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/3412f8705bc945388f5f43c941b22779 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/3412f8705bc945388f5f43c941b22779 2024-11-08T02:23:54,469 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/3412f8705bc945388f5f43c941b22779, entries=150, sequenceid=366, filesize=12.0 K 2024-11-08T02:23:54,470 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1dba34e1c528fab2e73d505d0e433dae in 885ms, sequenceid=366, compaction requested=true 2024-11-08T02:23:54,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:54,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:54,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-08T02:23:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-08T02:23:54,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-08T02:23:54,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0400 sec 2024-11-08T02:23:54,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0470 sec 2024-11-08T02:23:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T02:23:54,569 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:54,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-08T02:23:54,571 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T02:23:54,571 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:54,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T02:23:54,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-08T02:23:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:54,723 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:23:54,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/761a4acc53f5437abb469f2c6e995f3b is 50, key is test_row_0/A:col10/1731032633655/Put/seqid=0 2024-11-08T02:23:54,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742139_1315 (size=12301) 2024-11-08T02:23:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:54,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:54,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032694816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:54,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032694820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T02:23:54,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032694927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032694927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:54,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:55,139 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/761a4acc53f5437abb469f2c6e995f3b 2024-11-08T02:23:55,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032695138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:55,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032695138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:55,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/25a20cfead0e48c19197d5108db6d1a3 is 50, key is test_row_0/B:col10/1731032633655/Put/seqid=0 2024-11-08T02:23:55,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742140_1316 (size=12301) 2024-11-08T02:23:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T02:23:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032695448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032695448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:55,557 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/25a20cfead0e48c19197d5108db6d1a3 2024-11-08T02:23:55,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/cd8e3b061b194590a699fab2db583956 is 50, key is test_row_0/C:col10/1731032633655/Put/seqid=0 2024-11-08T02:23:55,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742141_1317 (size=12301) 2024-11-08T02:23:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T02:23:55,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032695958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032695958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:55,975 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/cd8e3b061b194590a699fab2db583956 2024-11-08T02:23:55,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/761a4acc53f5437abb469f2c6e995f3b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/761a4acc53f5437abb469f2c6e995f3b 2024-11-08T02:23:55,983 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/761a4acc53f5437abb469f2c6e995f3b, entries=150, sequenceid=390, filesize=12.0 K 2024-11-08T02:23:55,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/25a20cfead0e48c19197d5108db6d1a3 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/25a20cfead0e48c19197d5108db6d1a3 2024-11-08T02:23:55,995 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/25a20cfead0e48c19197d5108db6d1a3, entries=150, sequenceid=390, filesize=12.0 K 2024-11-08T02:23:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/cd8e3b061b194590a699fab2db583956 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/cd8e3b061b194590a699fab2db583956 2024-11-08T02:23:56,008 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/cd8e3b061b194590a699fab2db583956, entries=150, sequenceid=390, filesize=12.0 K 2024-11-08T02:23:56,009 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1dba34e1c528fab2e73d505d0e433dae in 1286ms, sequenceid=390, compaction requested=true 2024-11-08T02:23:56,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:56,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:56,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-08T02:23:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-08T02:23:56,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-08T02:23:56,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4390 sec 2024-11-08T02:23:56,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4430 sec 2024-11-08T02:23:56,570 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:23:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T02:23:56,718 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:56,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-08T02:23:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T02:23:56,720 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:56,721 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:56,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T02:23:56,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-08T02:23:56,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:56,873 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:23:56,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/30419b4a025e4ddaac2481d2ff489afb is 50, key is test_row_0/A:col10/1731032634814/Put/seqid=0 2024-11-08T02:23:56,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742142_1318 (size=12301) 2024-11-08T02:23:56,887 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/30419b4a025e4ddaac2481d2ff489afb 2024-11-08T02:23:56,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/dcede7009cce4177b69c0b616b798222 is 50, key is test_row_0/B:col10/1731032634814/Put/seqid=0 2024-11-08T02:23:56,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742143_1319 (size=12301) 2024-11-08T02:23:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:56,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:57,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032697032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T02:23:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032697036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:57,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032697148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:57,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:57,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032697148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:57,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:57,313 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/dcede7009cce4177b69c0b616b798222 2024-11-08T02:23:57,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/d71300eea8af435aa9e806e9ba39ba4c is 50, key is test_row_0/C:col10/1731032634814/Put/seqid=0 2024-11-08T02:23:57,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742144_1320 (size=12301) 2024-11-08T02:23:57,333 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/d71300eea8af435aa9e806e9ba39ba4c 2024-11-08T02:23:57,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/30419b4a025e4ddaac2481d2ff489afb as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/30419b4a025e4ddaac2481d2ff489afb 2024-11-08T02:23:57,345 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/30419b4a025e4ddaac2481d2ff489afb, entries=150, sequenceid=402, filesize=12.0 K 2024-11-08T02:23:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/dcede7009cce4177b69c0b616b798222 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dcede7009cce4177b69c0b616b798222 2024-11-08T02:23:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T02:23:57,351 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dcede7009cce4177b69c0b616b798222, entries=150, sequenceid=402, filesize=12.0 K 2024-11-08T02:23:57,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/d71300eea8af435aa9e806e9ba39ba4c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d71300eea8af435aa9e806e9ba39ba4c 2024-11-08T02:23:57,359 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d71300eea8af435aa9e806e9ba39ba4c, entries=150, sequenceid=402, filesize=12.0 K 2024-11-08T02:23:57,360 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1dba34e1c528fab2e73d505d0e433dae in 487ms, sequenceid=402, compaction requested=true 2024-11-08T02:23:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-08T02:23:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-08T02:23:57,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-08T02:23:57,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 640 msec 2024-11-08T02:23:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:57,363 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:23:57,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:23:57,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 644 msec 2024-11-08T02:23:57,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/33ece3829dad4293acf809896a9efec1 is 50, key is test_row_0/A:col10/1731032637035/Put/seqid=0 2024-11-08T02:23:57,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742145_1321 (size=17181) 2024-11-08T02:23:57,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032697381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:57,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032697385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032697488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:57,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032697498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:57,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032697697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:57,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032697708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:57,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/33ece3829dad4293acf809896a9efec1 2024-11-08T02:23:57,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/af200ce80efb4c44935f2a99488b222f is 50, key is test_row_0/B:col10/1731032637035/Put/seqid=0 2024-11-08T02:23:57,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742146_1322 (size=12301) 2024-11-08T02:23:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T02:23:57,858 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:23:57,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:23:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-08T02:23:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T02:23:57,860 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:23:57,861 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:23:57,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:23:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T02:23:58,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:58,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032698008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:58,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:58,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:58,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:58,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-08T02:23:58,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:58,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:58,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032698017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:58,090 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-08T02:23:58,090 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:58,090 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,090 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:58,091 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-11-08T02:23:58,091 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:58,091 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,091 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,091 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:58,092 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-08T02:23:58,092 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:58,092 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,092 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:58,093 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-08T02:23:58,093 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:58,093 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,093 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:58,097 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-08T02:23:58,097 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:58,097 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,097 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,097 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:58,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-08T02:23:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T02:23:58,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/af200ce80efb4c44935f2a99488b222f 2024-11-08T02:23:58,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/16937b4938084d338b4aab152dc2fab2 is 50, key is test_row_0/C:col10/1731032637035/Put/seqid=0 2024-11-08T02:23:58,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742147_1323 (size=12301) 2024-11-08T02:23:58,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-08T02:23:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-08T02:23:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:23:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T02:23:58,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39238 deadline: 1731032698518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:58,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:58,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:58,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:58,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:23:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39306 deadline: 1731032698528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 2024-11-08T02:23:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:23:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:23:58,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/16937b4938084d338b4aab152dc2fab2 2024-11-08T02:23:58,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/33ece3829dad4293acf809896a9efec1 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/33ece3829dad4293acf809896a9efec1 2024-11-08T02:23:58,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/33ece3829dad4293acf809896a9efec1, entries=250, sequenceid=428, filesize=16.8 K 2024-11-08T02:23:58,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/af200ce80efb4c44935f2a99488b222f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/af200ce80efb4c44935f2a99488b222f 2024-11-08T02:23:58,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/af200ce80efb4c44935f2a99488b222f, entries=150, sequenceid=428, filesize=12.0 K 2024-11-08T02:23:58,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/16937b4938084d338b4aab152dc2fab2 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/16937b4938084d338b4aab152dc2fab2 2024-11-08T02:23:58,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/16937b4938084d338b4aab152dc2fab2, entries=150, sequenceid=428, filesize=12.0 K 2024-11-08T02:23:58,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1dba34e1c528fab2e73d505d0e433dae in 1252ms, sequenceid=428, compaction requested=true 2024-11-08T02:23:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:23:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:58,615 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-08T02:23:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:23:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:58,615 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-08T02:23:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1dba34e1c528fab2e73d505d0e433dae:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:23:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:58,617 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86823 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-08T02:23:58,617 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/B is initiating minor compaction (all files) 2024-11-08T02:23:58,617 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 91703 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-08T02:23:58,617 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/B in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,617 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/A is initiating minor compaction (all files) 2024-11-08T02:23:58,617 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/A in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,617 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/005380dd31e7497ab1f8caf2ea442ef4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f40137d79ff54399a3bdf3ae93339bcf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0f569f59afae4eaaab6a2bc68c7273a4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e4b605b283c94135a3b9955bcce2efe0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/761a4acc53f5437abb469f2c6e995f3b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/30419b4a025e4ddaac2481d2ff489afb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/33ece3829dad4293acf809896a9efec1] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=89.6 K 2024-11-08T02:23:58,617 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/7fcf946d9b4f49b89009b5bccf35b71a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9d7eb8e77b4d4cc38acc327d4159aa89, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/aa47c3bc6f8c4a6a8ea54cf33afad9b5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9cf409db5380465180ce188af728308a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/25a20cfead0e48c19197d5108db6d1a3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dcede7009cce4177b69c0b616b798222, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/af200ce80efb4c44935f2a99488b222f] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=84.8 K 2024-11-08T02:23:58,617 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 005380dd31e7497ab1f8caf2ea442ef4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731032629279 2024-11-08T02:23:58,617 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fcf946d9b4f49b89009b5bccf35b71a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731032629279 2024-11-08T02:23:58,617 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting f40137d79ff54399a3bdf3ae93339bcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731032629927 2024-11-08T02:23:58,617 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d7eb8e77b4d4cc38acc327d4159aa89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731032629927 2024-11-08T02:23:58,618 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f569f59afae4eaaab6a2bc68c7273a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731032632123 2024-11-08T02:23:58,618 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting aa47c3bc6f8c4a6a8ea54cf33afad9b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731032632123 2024-11-08T02:23:58,618 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4b605b283c94135a3b9955bcce2efe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731032632462 2024-11-08T02:23:58,618 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf409db5380465180ce188af728308a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731032632462 2024-11-08T02:23:58,618 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761a4acc53f5437abb469f2c6e995f3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731032633641 2024-11-08T02:23:58,618 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 25a20cfead0e48c19197d5108db6d1a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731032633641 2024-11-08T02:23:58,618 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30419b4a025e4ddaac2481d2ff489afb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731032634810 2024-11-08T02:23:58,618 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting dcede7009cce4177b69c0b616b798222, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731032634810 2024-11-08T02:23:58,619 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33ece3829dad4293acf809896a9efec1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731032637026 2024-11-08T02:23:58,619 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting af200ce80efb4c44935f2a99488b222f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731032637026 2024-11-08T02:23:58,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-08T02:23:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,625 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T02:23:58,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/57b41631ea134661814cb072c8c26aa2 is 50, key is test_row_0/A:col10/1731032637384/Put/seqid=0 2024-11-08T02:23:58,637 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#B#compaction#283 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:58,638 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/a024e1986da24ca491e3148c300b6fb2 is 50, key is test_row_0/B:col10/1731032637035/Put/seqid=0 2024-11-08T02:23:58,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742148_1324 (size=12301) 2024-11-08T02:23:58,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742149_1325 (size=13255) 2024-11-08T02:23:58,654 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#A#compaction#284 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:58,655 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/878092db8dfc46438fe69727b550ddeb is 50, key is test_row_0/A:col10/1731032637035/Put/seqid=0 2024-11-08T02:23:58,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742150_1326 (size=13255) 2024-11-08T02:23:58,664 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/878092db8dfc46438fe69727b550ddeb as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/878092db8dfc46438fe69727b550ddeb 2024-11-08T02:23:58,669 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/A of 1dba34e1c528fab2e73d505d0e433dae into 878092db8dfc46438fe69727b550ddeb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:58,669 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:58,669 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/A, priority=9, startTime=1731032638615; duration=0sec 2024-11-08T02:23:58,669 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:23:58,669 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:A 2024-11-08T02:23:58,669 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-08T02:23:58,671 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86823 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-08T02:23:58,671 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 1dba34e1c528fab2e73d505d0e433dae/C is initiating minor compaction (all files) 2024-11-08T02:23:58,671 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1dba34e1c528fab2e73d505d0e433dae/C in TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:58,671 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/3d95cdec8abe451d81d426d639aea9a9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d174a893d2694174846893fdf7311453, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/37f5ad237b454e9687f86507d0b3612e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/3412f8705bc945388f5f43c941b22779, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/cd8e3b061b194590a699fab2db583956, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d71300eea8af435aa9e806e9ba39ba4c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/16937b4938084d338b4aab152dc2fab2] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp, totalSize=84.8 K 2024-11-08T02:23:58,671 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d95cdec8abe451d81d426d639aea9a9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731032629279 2024-11-08T02:23:58,672 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting d174a893d2694174846893fdf7311453, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731032629927 2024-11-08T02:23:58,672 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37f5ad237b454e9687f86507d0b3612e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731032632123 2024-11-08T02:23:58,672 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3412f8705bc945388f5f43c941b22779, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731032632462 2024-11-08T02:23:58,672 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd8e3b061b194590a699fab2db583956, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731032633641 2024-11-08T02:23:58,679 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting d71300eea8af435aa9e806e9ba39ba4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731032634810 2024-11-08T02:23:58,680 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16937b4938084d338b4aab152dc2fab2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731032637026 2024-11-08T02:23:58,696 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1dba34e1c528fab2e73d505d0e433dae#C#compaction#285 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:23:58,697 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/55b049aa040a48f5b44b2968df410966 is 50, key is test_row_0/C:col10/1731032637035/Put/seqid=0 2024-11-08T02:23:58,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742151_1327 (size=13255) 2024-11-08T02:23:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:23:58,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. as already flushing 2024-11-08T02:23:58,760 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-08T02:23:58,760 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:58,760 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,760 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:58,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T02:23:59,041 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/57b41631ea134661814cb072c8c26aa2 2024-11-08T02:23:59,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/1f1e826d70fe438b9a14f7395a2f23fd is 50, key is test_row_0/B:col10/1731032637384/Put/seqid=0 2024-11-08T02:23:59,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742152_1328 (size=12301) 2024-11-08T02:23:59,053 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/a024e1986da24ca491e3148c300b6fb2 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/a024e1986da24ca491e3148c300b6fb2 2024-11-08T02:23:59,058 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/B of 1dba34e1c528fab2e73d505d0e433dae into a024e1986da24ca491e3148c300b6fb2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:59,058 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:59,058 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/B, priority=9, startTime=1731032638615; duration=0sec 2024-11-08T02:23:59,058 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:59,058 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:B 2024-11-08T02:23:59,105 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/55b049aa040a48f5b44b2968df410966 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/55b049aa040a48f5b44b2968df410966 2024-11-08T02:23:59,110 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 1dba34e1c528fab2e73d505d0e433dae/C of 1dba34e1c528fab2e73d505d0e433dae into 55b049aa040a48f5b44b2968df410966(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:23:59,110 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:59,111 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., storeName=1dba34e1c528fab2e73d505d0e433dae/C, priority=9, startTime=1731032638615; duration=0sec 2024-11-08T02:23:59,111 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:23:59,111 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1dba34e1c528fab2e73d505d0e433dae:C 2024-11-08T02:23:59,451 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/1f1e826d70fe438b9a14f7395a2f23fd 2024-11-08T02:23:59,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/6ab283e99ba04d79a7876e5d41df71f8 is 50, key is test_row_0/C:col10/1731032637384/Put/seqid=0 2024-11-08T02:23:59,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742153_1329 (size=12301) 2024-11-08T02:23:59,529 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-11-08T02:23:59,529 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:59,529 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:59,529 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:59,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:59,539 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-08T02:23:59,539 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:23:59,539 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:59,540 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:23:59,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:23:59,861 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/6ab283e99ba04d79a7876e5d41df71f8 2024-11-08T02:23:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/57b41631ea134661814cb072c8c26aa2 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/57b41631ea134661814cb072c8c26aa2 2024-11-08T02:23:59,868 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/57b41631ea134661814cb072c8c26aa2, entries=150, sequenceid=438, filesize=12.0 K 2024-11-08T02:23:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/1f1e826d70fe438b9a14f7395a2f23fd as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/1f1e826d70fe438b9a14f7395a2f23fd 2024-11-08T02:23:59,872 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/1f1e826d70fe438b9a14f7395a2f23fd, entries=150, sequenceid=438, filesize=12.0 K 2024-11-08T02:23:59,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/6ab283e99ba04d79a7876e5d41df71f8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/6ab283e99ba04d79a7876e5d41df71f8 2024-11-08T02:23:59,877 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/6ab283e99ba04d79a7876e5d41df71f8, entries=150, sequenceid=438, filesize=12.0 K 2024-11-08T02:23:59,877 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=20.13 KB/20610 for 1dba34e1c528fab2e73d505d0e433dae in 1252ms, sequenceid=438, compaction requested=false 2024-11-08T02:23:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 1dba34e1c528fab2e73d505d0e433dae: 2024-11-08T02:23:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:23:59,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-08T02:23:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-08T02:23:59,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-08T02:23:59,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0180 sec 2024-11-08T02:23:59,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.0210 sec 2024-11-08T02:24:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T02:24:00,018 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:00,839 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-08T02:24:00,839 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:00,839 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:00,839 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:00,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:00,879 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-08T02:24:00,879 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:00,879 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:00,880 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:00,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 163 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3708 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3598 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3596 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3683 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3547 2024-11-08T02:24:00,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:24:00,880 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T02:24:00,880 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:24:00,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:00,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:00,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:00,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:24:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-08T02:24:00,883 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032640883"}]},"ts":"1731032640883"} 2024-11-08T02:24:00,884 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:24:00,884 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:24:00,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:24:00,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1dba34e1c528fab2e73d505d0e433dae, UNASSIGN}] 2024-11-08T02:24:00,886 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1dba34e1c528fab2e73d505d0e433dae, UNASSIGN 2024-11-08T02:24:00,887 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=1dba34e1c528fab2e73d505d0e433dae, regionState=CLOSING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:24:00,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1dba34e1c528fab2e73d505d0e433dae, UNASSIGN because future has completed 2024-11-08T02:24:00,888 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:24:00,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:24:00,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-08T02:24:01,040 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:24:01,041 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T02:24:01,041 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 1dba34e1c528fab2e73d505d0e433dae, disabling compactions & flushes 2024-11-08T02:24:01,041 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:24:01,041 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:24:01,041 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. after waiting 0 ms 2024-11-08T02:24:01,041 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:24:01,041 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 1dba34e1c528fab2e73d505d0e433dae 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T02:24:01,044 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/b6e842ee4235433ebfa8d7170ae970be is 50, key is test_row_0/A:col10/1731032638758/Put/seqid=0 2024-11-08T02:24:01,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742154_1330 (size=12301) 2024-11-08T02:24:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-08T02:24:01,447 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/b6e842ee4235433ebfa8d7170ae970be 2024-11-08T02:24:01,454 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/158cb8fecb564130b2a677a0e0bc2087 is 50, key is test_row_0/B:col10/1731032638758/Put/seqid=0 2024-11-08T02:24:01,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742155_1331 (size=12301) 2024-11-08T02:24:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-08T02:24:01,858 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/158cb8fecb564130b2a677a0e0bc2087 2024-11-08T02:24:01,865 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/73dd4798b2964593bf6ad93965c905aa is 50, key is test_row_0/C:col10/1731032638758/Put/seqid=0 2024-11-08T02:24:01,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742156_1332 (size=12301) 2024-11-08T02:24:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-08T02:24:02,269 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/73dd4798b2964593bf6ad93965c905aa 2024-11-08T02:24:02,274 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/A/b6e842ee4235433ebfa8d7170ae970be as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b6e842ee4235433ebfa8d7170ae970be 2024-11-08T02:24:02,278 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b6e842ee4235433ebfa8d7170ae970be, entries=150, sequenceid=449, filesize=12.0 K 2024-11-08T02:24:02,279 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/B/158cb8fecb564130b2a677a0e0bc2087 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/158cb8fecb564130b2a677a0e0bc2087 2024-11-08T02:24:02,282 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/158cb8fecb564130b2a677a0e0bc2087, entries=150, sequenceid=449, filesize=12.0 K 2024-11-08T02:24:02,283 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/.tmp/C/73dd4798b2964593bf6ad93965c905aa as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/73dd4798b2964593bf6ad93965c905aa 2024-11-08T02:24:02,286 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/73dd4798b2964593bf6ad93965c905aa, entries=150, sequenceid=449, filesize=12.0 K 2024-11-08T02:24:02,287 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1dba34e1c528fab2e73d505d0e433dae in 1245ms, sequenceid=449, compaction requested=true 2024-11-08T02:24:02,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/df4c709337224cd3a829579559b53444, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/085460b9408f4da39cc6cc42e09b572c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/fad699a41d934a309ea2e6cbaf8b6e64, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f59acb5586c14e0d98dcb27fce5ed93f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/9977d5e9ee4f44efaf3e8973bb2433cf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b36fc6ad594b4a98adf65e2fb366b27d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c39266773c404dc99202cf56f1d9aa58, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/fc230ea8a2714dd5b7dfb369269219a1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c6ee39ad42e840119401c14328062871, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e3df008adf6a45a6a296cace9925a721, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/a7847bea20794bcca59250b066805e13, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/052ee0a0503e4921aae5113848b91526, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/8571b5f8d74e44d899a86600bbbd03cd, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/802146d4b33846229451b2ec8e2ea39b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/6f1c64870022424db04ffd6a71f947ce, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0180e8b3d7af4cb2b57e8522cfef5541, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/6a245d3d4356414c90613a8863091171, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/9784ffee8edf4344884f0ed5efa1fc64, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c1b3bce427ea42d2bd021a6b95ac7b55, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c3a51134c43a4c978d02a8661028fbf2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b5a3236f1ac04a2e90c77bf2446230af, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/005380dd31e7497ab1f8caf2ea442ef4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f40137d79ff54399a3bdf3ae93339bcf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0f569f59afae4eaaab6a2bc68c7273a4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e4b605b283c94135a3b9955bcce2efe0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/761a4acc53f5437abb469f2c6e995f3b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/30419b4a025e4ddaac2481d2ff489afb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/33ece3829dad4293acf809896a9efec1] to archive 2024-11-08T02:24:02,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:24:02,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/df4c709337224cd3a829579559b53444 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/df4c709337224cd3a829579559b53444 2024-11-08T02:24:02,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/085460b9408f4da39cc6cc42e09b572c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/085460b9408f4da39cc6cc42e09b572c 2024-11-08T02:24:02,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/fad699a41d934a309ea2e6cbaf8b6e64 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/fad699a41d934a309ea2e6cbaf8b6e64 2024-11-08T02:24:02,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f59acb5586c14e0d98dcb27fce5ed93f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f59acb5586c14e0d98dcb27fce5ed93f 2024-11-08T02:24:02,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/9977d5e9ee4f44efaf3e8973bb2433cf to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/9977d5e9ee4f44efaf3e8973bb2433cf 2024-11-08T02:24:02,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b36fc6ad594b4a98adf65e2fb366b27d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b36fc6ad594b4a98adf65e2fb366b27d 2024-11-08T02:24:02,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c39266773c404dc99202cf56f1d9aa58 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c39266773c404dc99202cf56f1d9aa58 2024-11-08T02:24:02,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/fc230ea8a2714dd5b7dfb369269219a1 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/fc230ea8a2714dd5b7dfb369269219a1 2024-11-08T02:24:02,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c6ee39ad42e840119401c14328062871 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c6ee39ad42e840119401c14328062871 2024-11-08T02:24:02,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e3df008adf6a45a6a296cace9925a721 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e3df008adf6a45a6a296cace9925a721 2024-11-08T02:24:02,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/a7847bea20794bcca59250b066805e13 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/a7847bea20794bcca59250b066805e13 2024-11-08T02:24:02,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/052ee0a0503e4921aae5113848b91526 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/052ee0a0503e4921aae5113848b91526 2024-11-08T02:24:02,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/8571b5f8d74e44d899a86600bbbd03cd to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/8571b5f8d74e44d899a86600bbbd03cd 2024-11-08T02:24:02,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/802146d4b33846229451b2ec8e2ea39b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/802146d4b33846229451b2ec8e2ea39b 2024-11-08T02:24:02,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/6f1c64870022424db04ffd6a71f947ce to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/6f1c64870022424db04ffd6a71f947ce 2024-11-08T02:24:02,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0180e8b3d7af4cb2b57e8522cfef5541 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0180e8b3d7af4cb2b57e8522cfef5541 2024-11-08T02:24:02,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/6a245d3d4356414c90613a8863091171 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/6a245d3d4356414c90613a8863091171 2024-11-08T02:24:02,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/9784ffee8edf4344884f0ed5efa1fc64 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/9784ffee8edf4344884f0ed5efa1fc64 2024-11-08T02:24:02,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c1b3bce427ea42d2bd021a6b95ac7b55 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c1b3bce427ea42d2bd021a6b95ac7b55 2024-11-08T02:24:02,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c3a51134c43a4c978d02a8661028fbf2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/c3a51134c43a4c978d02a8661028fbf2 2024-11-08T02:24:02,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b5a3236f1ac04a2e90c77bf2446230af to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b5a3236f1ac04a2e90c77bf2446230af 2024-11-08T02:24:02,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/005380dd31e7497ab1f8caf2ea442ef4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/005380dd31e7497ab1f8caf2ea442ef4 2024-11-08T02:24:02,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f40137d79ff54399a3bdf3ae93339bcf to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/f40137d79ff54399a3bdf3ae93339bcf 2024-11-08T02:24:02,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0f569f59afae4eaaab6a2bc68c7273a4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/0f569f59afae4eaaab6a2bc68c7273a4 2024-11-08T02:24:02,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e4b605b283c94135a3b9955bcce2efe0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/e4b605b283c94135a3b9955bcce2efe0 2024-11-08T02:24:02,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/761a4acc53f5437abb469f2c6e995f3b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/761a4acc53f5437abb469f2c6e995f3b 2024-11-08T02:24:02,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/30419b4a025e4ddaac2481d2ff489afb to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/30419b4a025e4ddaac2481d2ff489afb 2024-11-08T02:24:02,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/33ece3829dad4293acf809896a9efec1 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/33ece3829dad4293acf809896a9efec1 2024-11-08T02:24:02,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dee419b3140a480087326dae0f1d84f6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5f9856fcdb9a4a0fa4d4b205ed56b241, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/572343c2841f4a4d9e33390056d2f4bb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/16fd880921264c12b00244cb755f0f34, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/48f66f8bd0ab4e9bbcdf116e4fa0927c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/f4c4eeee599143a1991ef9936b4d1768, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/854cef1a87c141aca4c94fd80f6ae176, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/4f11b23f7cf2456db971206320a5898c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/e5fff8b66e8348ceaff5e768046f0606, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/02eb53cf49b34c889aa1cd0f9b490b0f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/6cddea1685a74ae0b18cdcd67efee628, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5266b6b7e82e49f4b4c01ae5e096847b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/df5341bcd9094fe1b514c3bfd05b8515, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/c5dac2d1d9c04b9791b90287b405a54e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/573bb5081959448eba3b2cb1b6e0508a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/83ff99db190a42a5a80fef82bba2f4f7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/535700f7129843659216975b3d6e2583, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/3e25751b205f4a969a3dbe2a9f76eba7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/05f3affd922d4e8185186acabe652cb4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b908207c396446ff8e9ef6815cd48a0e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/7fcf946d9b4f49b89009b5bccf35b71a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b56dab8bcea2411ca148d5099a79386a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9d7eb8e77b4d4cc38acc327d4159aa89, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/aa47c3bc6f8c4a6a8ea54cf33afad9b5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9cf409db5380465180ce188af728308a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/25a20cfead0e48c19197d5108db6d1a3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dcede7009cce4177b69c0b616b798222, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/af200ce80efb4c44935f2a99488b222f] to archive 2024-11-08T02:24:02,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:24:02,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dee419b3140a480087326dae0f1d84f6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dee419b3140a480087326dae0f1d84f6 2024-11-08T02:24:02,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5f9856fcdb9a4a0fa4d4b205ed56b241 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5f9856fcdb9a4a0fa4d4b205ed56b241 2024-11-08T02:24:02,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/572343c2841f4a4d9e33390056d2f4bb to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/572343c2841f4a4d9e33390056d2f4bb 2024-11-08T02:24:02,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/16fd880921264c12b00244cb755f0f34 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/16fd880921264c12b00244cb755f0f34 2024-11-08T02:24:02,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/48f66f8bd0ab4e9bbcdf116e4fa0927c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/48f66f8bd0ab4e9bbcdf116e4fa0927c 2024-11-08T02:24:02,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/f4c4eeee599143a1991ef9936b4d1768 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/f4c4eeee599143a1991ef9936b4d1768 2024-11-08T02:24:02,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/854cef1a87c141aca4c94fd80f6ae176 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/854cef1a87c141aca4c94fd80f6ae176 2024-11-08T02:24:02,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/4f11b23f7cf2456db971206320a5898c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/4f11b23f7cf2456db971206320a5898c 2024-11-08T02:24:02,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/e5fff8b66e8348ceaff5e768046f0606 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/e5fff8b66e8348ceaff5e768046f0606 2024-11-08T02:24:02,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/02eb53cf49b34c889aa1cd0f9b490b0f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/02eb53cf49b34c889aa1cd0f9b490b0f 2024-11-08T02:24:02,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/6cddea1685a74ae0b18cdcd67efee628 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/6cddea1685a74ae0b18cdcd67efee628 2024-11-08T02:24:02,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5266b6b7e82e49f4b4c01ae5e096847b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/5266b6b7e82e49f4b4c01ae5e096847b 2024-11-08T02:24:02,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/df5341bcd9094fe1b514c3bfd05b8515 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/df5341bcd9094fe1b514c3bfd05b8515 2024-11-08T02:24:02,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/c5dac2d1d9c04b9791b90287b405a54e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/c5dac2d1d9c04b9791b90287b405a54e 2024-11-08T02:24:02,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/573bb5081959448eba3b2cb1b6e0508a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/573bb5081959448eba3b2cb1b6e0508a 2024-11-08T02:24:02,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/83ff99db190a42a5a80fef82bba2f4f7 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/83ff99db190a42a5a80fef82bba2f4f7 2024-11-08T02:24:02,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/535700f7129843659216975b3d6e2583 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/535700f7129843659216975b3d6e2583 2024-11-08T02:24:02,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/3e25751b205f4a969a3dbe2a9f76eba7 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/3e25751b205f4a969a3dbe2a9f76eba7 2024-11-08T02:24:02,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/05f3affd922d4e8185186acabe652cb4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/05f3affd922d4e8185186acabe652cb4 2024-11-08T02:24:02,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b908207c396446ff8e9ef6815cd48a0e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b908207c396446ff8e9ef6815cd48a0e 2024-11-08T02:24:02,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/7fcf946d9b4f49b89009b5bccf35b71a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/7fcf946d9b4f49b89009b5bccf35b71a 2024-11-08T02:24:02,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b56dab8bcea2411ca148d5099a79386a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/b56dab8bcea2411ca148d5099a79386a 2024-11-08T02:24:02,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9d7eb8e77b4d4cc38acc327d4159aa89 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9d7eb8e77b4d4cc38acc327d4159aa89 2024-11-08T02:24:02,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/aa47c3bc6f8c4a6a8ea54cf33afad9b5 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/aa47c3bc6f8c4a6a8ea54cf33afad9b5 2024-11-08T02:24:02,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9cf409db5380465180ce188af728308a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/9cf409db5380465180ce188af728308a 2024-11-08T02:24:02,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/25a20cfead0e48c19197d5108db6d1a3 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/25a20cfead0e48c19197d5108db6d1a3 2024-11-08T02:24:02,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dcede7009cce4177b69c0b616b798222 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/dcede7009cce4177b69c0b616b798222 2024-11-08T02:24:02,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/af200ce80efb4c44935f2a99488b222f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/af200ce80efb4c44935f2a99488b222f 2024-11-08T02:24:02,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/657ce7b4959446888c64af5a9b922efe, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/9d003ced8bf543b189cc0c28c5efda39, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/7eca0758584d43d0ae5316c118e21d2a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/0ec3c52b2d4d48e6aaaa80ad290f9870, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/2daf9bb9ab7240e3bd19c4f755a2f8e3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a2f14dbfff1c4b1ea1d5b4e606b6e0d8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/cfbf5b268340439d9bd9505bbb63df41, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/79f04cb81fd349b2afc204e92af5e751, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/14675684861746769622821eb19b3ea9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/52f4f0aa5c684b4e8a6ef3836bc5500a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/dc229689424549b2ac7019aabac28ba8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/80659acd29044bb392402a207f335523, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/76e6514ca0d44e52a334af2e7132b563, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/6ce71e007dc2439abfba83d952a0dec1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/756fd5915843455f9731a2a9fed3b490, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/c757dc3bd7814db79945c65fc2305b94, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/5db0aafca1054a5a8706c72f80ca5af1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a5a8255abccd484f977cdc9b1936300b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a56505f79f6449f3be729ac2be6e066a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/8d00a73473c2419292332b669f7f05e7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/3d95cdec8abe451d81d426d639aea9a9, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/e18447cbb8d64056921d761b2217b269, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d174a893d2694174846893fdf7311453, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/37f5ad237b454e9687f86507d0b3612e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/3412f8705bc945388f5f43c941b22779, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/cd8e3b061b194590a699fab2db583956, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d71300eea8af435aa9e806e9ba39ba4c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/16937b4938084d338b4aab152dc2fab2] to archive 2024-11-08T02:24:02,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:24:02,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/657ce7b4959446888c64af5a9b922efe to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/657ce7b4959446888c64af5a9b922efe 2024-11-08T02:24:02,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/9d003ced8bf543b189cc0c28c5efda39 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/9d003ced8bf543b189cc0c28c5efda39 2024-11-08T02:24:02,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/7eca0758584d43d0ae5316c118e21d2a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/7eca0758584d43d0ae5316c118e21d2a 2024-11-08T02:24:02,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/0ec3c52b2d4d48e6aaaa80ad290f9870 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/0ec3c52b2d4d48e6aaaa80ad290f9870 2024-11-08T02:24:02,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/2daf9bb9ab7240e3bd19c4f755a2f8e3 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/2daf9bb9ab7240e3bd19c4f755a2f8e3 2024-11-08T02:24:02,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a2f14dbfff1c4b1ea1d5b4e606b6e0d8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a2f14dbfff1c4b1ea1d5b4e606b6e0d8 2024-11-08T02:24:02,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/cfbf5b268340439d9bd9505bbb63df41 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/cfbf5b268340439d9bd9505bbb63df41 2024-11-08T02:24:02,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/79f04cb81fd349b2afc204e92af5e751 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/79f04cb81fd349b2afc204e92af5e751 2024-11-08T02:24:02,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/14675684861746769622821eb19b3ea9 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/14675684861746769622821eb19b3ea9 2024-11-08T02:24:02,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/52f4f0aa5c684b4e8a6ef3836bc5500a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/52f4f0aa5c684b4e8a6ef3836bc5500a 2024-11-08T02:24:02,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/dc229689424549b2ac7019aabac28ba8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/dc229689424549b2ac7019aabac28ba8 2024-11-08T02:24:02,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/80659acd29044bb392402a207f335523 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/80659acd29044bb392402a207f335523 2024-11-08T02:24:02,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/76e6514ca0d44e52a334af2e7132b563 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/76e6514ca0d44e52a334af2e7132b563 2024-11-08T02:24:02,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/6ce71e007dc2439abfba83d952a0dec1 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/6ce71e007dc2439abfba83d952a0dec1 2024-11-08T02:24:02,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/756fd5915843455f9731a2a9fed3b490 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/756fd5915843455f9731a2a9fed3b490 2024-11-08T02:24:02,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/c757dc3bd7814db79945c65fc2305b94 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/c757dc3bd7814db79945c65fc2305b94 2024-11-08T02:24:02,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/5db0aafca1054a5a8706c72f80ca5af1 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/5db0aafca1054a5a8706c72f80ca5af1 2024-11-08T02:24:02,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a5a8255abccd484f977cdc9b1936300b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a5a8255abccd484f977cdc9b1936300b 2024-11-08T02:24:02,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a56505f79f6449f3be729ac2be6e066a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/a56505f79f6449f3be729ac2be6e066a 2024-11-08T02:24:02,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/8d00a73473c2419292332b669f7f05e7 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/8d00a73473c2419292332b669f7f05e7 2024-11-08T02:24:02,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/3d95cdec8abe451d81d426d639aea9a9 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/3d95cdec8abe451d81d426d639aea9a9 2024-11-08T02:24:02,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/e18447cbb8d64056921d761b2217b269 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/e18447cbb8d64056921d761b2217b269 2024-11-08T02:24:02,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d174a893d2694174846893fdf7311453 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d174a893d2694174846893fdf7311453 2024-11-08T02:24:02,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/37f5ad237b454e9687f86507d0b3612e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/37f5ad237b454e9687f86507d0b3612e 2024-11-08T02:24:02,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/3412f8705bc945388f5f43c941b22779 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/3412f8705bc945388f5f43c941b22779 2024-11-08T02:24:02,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/cd8e3b061b194590a699fab2db583956 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/cd8e3b061b194590a699fab2db583956 2024-11-08T02:24:02,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d71300eea8af435aa9e806e9ba39ba4c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/d71300eea8af435aa9e806e9ba39ba4c 2024-11-08T02:24:02,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/16937b4938084d338b4aab152dc2fab2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/16937b4938084d338b4aab152dc2fab2 2024-11-08T02:24:02,361 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/recovered.edits/452.seqid, newMaxSeqId=452, maxSeqId=1 2024-11-08T02:24:02,362 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. 2024-11-08T02:24:02,362 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 1dba34e1c528fab2e73d505d0e433dae: Waiting for close lock at 1731032641041Running coprocessor pre-close hooks at 1731032641041Disabling compacts and flushes for region at 1731032641041Disabling writes for close at 1731032641041Obtaining lock to block concurrent updates at 1731032641041Preparing flush snapshotting stores in 1dba34e1c528fab2e73d505d0e433dae at 1731032641041Finished memstore snapshotting TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731032641041Flushing stores of TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae. at 1731032641041Flushing 1dba34e1c528fab2e73d505d0e433dae/A: creating writer at 1731032641041Flushing 1dba34e1c528fab2e73d505d0e433dae/A: appending metadata at 1731032641044 (+3 ms)Flushing 1dba34e1c528fab2e73d505d0e433dae/A: closing flushed file at 1731032641044Flushing 1dba34e1c528fab2e73d505d0e433dae/B: creating writer at 1731032641452 (+408 ms)Flushing 1dba34e1c528fab2e73d505d0e433dae/B: appending metadata at 1731032641454 (+2 ms)Flushing 1dba34e1c528fab2e73d505d0e433dae/B: closing flushed file at 1731032641454Flushing 1dba34e1c528fab2e73d505d0e433dae/C: creating writer at 1731032641862 (+408 ms)Flushing 1dba34e1c528fab2e73d505d0e433dae/C: appending metadata at 1731032641864 (+2 ms)Flushing 1dba34e1c528fab2e73d505d0e433dae/C: closing flushed file at 1731032641864Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23cd6c4b: reopening flushed file at 1731032642273 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a1256f7: reopening flushed file at 1731032642278 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dd8cc4f: reopening flushed file at 1731032642282 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 1dba34e1c528fab2e73d505d0e433dae in 1245ms, sequenceid=449, compaction requested=true at 1731032642287 (+5 ms)Writing region close event to WAL at 1731032642359 (+72 ms)Running coprocessor post-close hooks at 1731032642362 (+3 ms)Closed at 1731032642362 2024-11-08T02:24:02,364 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:24:02,364 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=1dba34e1c528fab2e73d505d0e433dae, regionState=CLOSED 2024-11-08T02:24:02,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:24:02,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-08T02:24:02,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 1dba34e1c528fab2e73d505d0e433dae, server=331c7316141f,34325,1731032548147 in 1.4780 sec 2024-11-08T02:24:02,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-08T02:24:02,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1dba34e1c528fab2e73d505d0e433dae, UNASSIGN in 1.4810 sec 2024-11-08T02:24:02,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-08T02:24:02,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4840 sec 2024-11-08T02:24:02,371 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032642371"}]},"ts":"1731032642371"} 2024-11-08T02:24:02,372 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:24:02,372 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:24:02,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4920 sec 2024-11-08T02:24:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-08T02:24:03,028 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:03,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:24:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:03,030 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:03,031 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-08T02:24:03,032 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:24:03,034 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/recovered.edits] 2024-11-08T02:24:03,037 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/57b41631ea134661814cb072c8c26aa2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/57b41631ea134661814cb072c8c26aa2 2024-11-08T02:24:03,038 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/878092db8dfc46438fe69727b550ddeb to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/878092db8dfc46438fe69727b550ddeb 2024-11-08T02:24:03,039 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b6e842ee4235433ebfa8d7170ae970be to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/A/b6e842ee4235433ebfa8d7170ae970be 2024-11-08T02:24:03,041 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/158cb8fecb564130b2a677a0e0bc2087 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/158cb8fecb564130b2a677a0e0bc2087 2024-11-08T02:24:03,042 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/1f1e826d70fe438b9a14f7395a2f23fd to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/1f1e826d70fe438b9a14f7395a2f23fd 2024-11-08T02:24:03,043 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/a024e1986da24ca491e3148c300b6fb2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/B/a024e1986da24ca491e3148c300b6fb2 2024-11-08T02:24:03,044 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/55b049aa040a48f5b44b2968df410966 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/55b049aa040a48f5b44b2968df410966 2024-11-08T02:24:03,045 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/6ab283e99ba04d79a7876e5d41df71f8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/6ab283e99ba04d79a7876e5d41df71f8 2024-11-08T02:24:03,046 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/73dd4798b2964593bf6ad93965c905aa to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/C/73dd4798b2964593bf6ad93965c905aa 2024-11-08T02:24:03,048 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/recovered.edits/452.seqid to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae/recovered.edits/452.seqid 2024-11-08T02:24:03,049 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/1dba34e1c528fab2e73d505d0e433dae 2024-11-08T02:24:03,049 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:24:03,051 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:03,053 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:24:03,054 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:24:03,055 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:03,055 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:24:03,055 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731032643055"}]},"ts":"9223372036854775807"} 2024-11-08T02:24:03,057 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T02:24:03,057 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1dba34e1c528fab2e73d505d0e433dae, NAME => 'TestAcidGuarantees,,1731032615872.1dba34e1c528fab2e73d505d0e433dae.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:24:03,057 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:24:03,057 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731032643057"}]},"ts":"9223372036854775807"} 2024-11-08T02:24:03,059 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T02:24:03,060 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:03,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-08T02:24:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-08T02:24:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T02:24:03,138 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:03,146 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-403120256_22 at /127.0.0.1:49740 [Waiting for operation #362] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-548309475_22 at /127.0.0.1:49988 [Waiting for operation #211] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=546 (was 510) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6719 (was 6815) 2024-11-08T02:24:03,154 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=546, ProcessCount=11, AvailableMemoryMB=6719 2024-11-08T02:24:03,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:24:03,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:24:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:03,157 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:24:03,158 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:03,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-08T02:24:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T02:24:03,158 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:24:03,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742157_1333 (size=959) 2024-11-08T02:24:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T02:24:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T02:24:03,565 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee 2024-11-08T02:24:03,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742158_1334 (size=53) 2024-11-08T02:24:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T02:24:03,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:24:03,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a03465591aab9088b0f79bf5ef3a6e88, disabling compactions & flushes 2024-11-08T02:24:03,971 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:03,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:03,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. after waiting 0 ms 2024-11-08T02:24:03,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:03,971 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:03,971 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a03465591aab9088b0f79bf5ef3a6e88: Waiting for close lock at 1731032643971Disabling compacts and flushes for region at 1731032643971Disabling writes for close at 1731032643971Writing region close event to WAL at 1731032643971Closed at 1731032643971 2024-11-08T02:24:03,973 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:24:03,973 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731032643973"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731032643973"}]},"ts":"1731032643973"} 2024-11-08T02:24:03,974 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T02:24:03,975 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:24:03,975 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032643975"}]},"ts":"1731032643975"} 2024-11-08T02:24:03,977 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:24:03,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, ASSIGN}] 2024-11-08T02:24:03,978 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, ASSIGN 2024-11-08T02:24:03,979 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, ASSIGN; state=OFFLINE, location=331c7316141f,34325,1731032548147; forceNewPlan=false, retain=false 2024-11-08T02:24:04,129 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a03465591aab9088b0f79bf5ef3a6e88, regionState=OPENING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:24:04,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, ASSIGN because future has completed 2024-11-08T02:24:04,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:24:04,286 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:04,286 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:24:04,287 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,287 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:24:04,287 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,287 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,288 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,289 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a03465591aab9088b0f79bf5ef3a6e88 columnFamilyName A 2024-11-08T02:24:04,289 DEBUG [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:04,290 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(327): Store=a03465591aab9088b0f79bf5ef3a6e88/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:24:04,290 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,291 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a03465591aab9088b0f79bf5ef3a6e88 columnFamilyName B 2024-11-08T02:24:04,291 DEBUG [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:04,291 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(327): Store=a03465591aab9088b0f79bf5ef3a6e88/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:24:04,291 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,292 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a03465591aab9088b0f79bf5ef3a6e88 columnFamilyName C 2024-11-08T02:24:04,292 DEBUG [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:04,292 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(327): Store=a03465591aab9088b0f79bf5ef3a6e88/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:24:04,293 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,293 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,293 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,294 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,294 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,294 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:24:04,295 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,297 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:24:04,297 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened a03465591aab9088b0f79bf5ef3a6e88; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65797027, jitterRate=-0.019547894597053528}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:24:04,297 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T02:24:04,298 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for a03465591aab9088b0f79bf5ef3a6e88: Running coprocessor pre-open hook at 1731032644287Writing region info on filesystem at 1731032644287Initializing all the Stores at 1731032644288 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032644288Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032644288Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032644288Cleaning up temporary data from old regions at 1731032644294 (+6 ms)Running coprocessor post-open hooks at 1731032644297 (+3 ms)Region opened successfully at 1731032644298 (+1 ms) 2024-11-08T02:24:04,299 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., pid=91, masterSystemTime=1731032644283 2024-11-08T02:24:04,300 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:04,300 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:04,301 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=a03465591aab9088b0f79bf5ef3a6e88, regionState=OPEN, openSeqNum=2, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:24:04,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:24:04,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-08T02:24:04,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 in 171 msec 2024-11-08T02:24:04,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-08T02:24:04,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, ASSIGN in 327 msec 2024-11-08T02:24:04,306 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:24:04,306 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032644306"}]},"ts":"1731032644306"} 2024-11-08T02:24:04,307 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:24:04,308 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:24:04,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-08T02:24:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-08T02:24:05,308 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:05,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:05,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:05,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:05,310 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b63bd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:05,312 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:05,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7929de8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:05,313 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:05,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:05,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T02:24:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 331c7316141f,41605,1731032547397 2024-11-08T02:24:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6a6d0c21 2024-11-08T02:24:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:24:05,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:24:05,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:24:05,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:24:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:05,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742159_1335 (size=995) 2024-11-08T02:24:05,729 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T02:24:05,729 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T02:24:05,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:24:05,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, REOPEN/MOVE}] 2024-11-08T02:24:05,735 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, REOPEN/MOVE 2024-11-08T02:24:05,735 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a03465591aab9088b0f79bf5ef3a6e88, regionState=CLOSING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:24:05,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, REOPEN/MOVE because future has completed 2024-11-08T02:24:05,737 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:24:05,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:24:05,889 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:05,889 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T02:24:05,889 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing a03465591aab9088b0f79bf5ef3a6e88, disabling compactions & flushes 2024-11-08T02:24:05,889 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:05,889 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:05,889 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. after waiting 0 ms 2024-11-08T02:24:05,889 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:05,893 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T02:24:05,893 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:05,893 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for a03465591aab9088b0f79bf5ef3a6e88: Waiting for close lock at 1731032645889Running coprocessor pre-close hooks at 1731032645889Disabling compacts and flushes for region at 1731032645889Disabling writes for close at 1731032645889Writing region close event to WAL at 1731032645890 (+1 ms)Running coprocessor post-close hooks at 1731032645893 (+3 ms)Closed at 1731032645893 2024-11-08T02:24:05,893 WARN [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: a03465591aab9088b0f79bf5ef3a6e88 to self. 2024-11-08T02:24:05,895 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:05,896 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a03465591aab9088b0f79bf5ef3a6e88, regionState=CLOSED 2024-11-08T02:24:05,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:24:05,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-08T02:24:05,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 in 161 msec 2024-11-08T02:24:05,900 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, REOPEN/MOVE; state=CLOSED, location=331c7316141f,34325,1731032548147; forceNewPlan=false, retain=true 2024-11-08T02:24:06,050 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a03465591aab9088b0f79bf5ef3a6e88, regionState=OPENING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, REOPEN/MOVE because future has completed 2024-11-08T02:24:06,052 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:24:06,206 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:06,206 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:24:06,206 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,206 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:24:06,206 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,206 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,207 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,208 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a03465591aab9088b0f79bf5ef3a6e88 columnFamilyName A 2024-11-08T02:24:06,209 DEBUG [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:06,210 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(327): Store=a03465591aab9088b0f79bf5ef3a6e88/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:24:06,210 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,211 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a03465591aab9088b0f79bf5ef3a6e88 columnFamilyName B 2024-11-08T02:24:06,211 DEBUG [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:06,211 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(327): Store=a03465591aab9088b0f79bf5ef3a6e88/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:24:06,211 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,212 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a03465591aab9088b0f79bf5ef3a6e88 columnFamilyName C 2024-11-08T02:24:06,212 DEBUG [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:06,212 INFO [StoreOpener-a03465591aab9088b0f79bf5ef3a6e88-1 {}] regionserver.HStore(327): Store=a03465591aab9088b0f79bf5ef3a6e88/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:24:06,212 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,213 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,214 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,215 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,215 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,215 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:24:06,216 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,217 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened a03465591aab9088b0f79bf5ef3a6e88; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72872103, jitterRate=0.08587895333766937}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:24:06,217 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,218 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for a03465591aab9088b0f79bf5ef3a6e88: Running coprocessor pre-open hook at 1731032646206Writing region info on filesystem at 1731032646206Initializing all the Stores at 1731032646207 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032646207Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032646207Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032646207Cleaning up temporary data from old regions at 1731032646215 (+8 ms)Running coprocessor post-open hooks at 1731032646217 (+2 ms)Region opened successfully at 1731032646218 (+1 ms) 2024-11-08T02:24:06,219 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., pid=96, masterSystemTime=1731032646203 2024-11-08T02:24:06,220 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:06,220 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:06,221 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=a03465591aab9088b0f79bf5ef3a6e88, regionState=OPEN, openSeqNum=5, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:24:06,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-08T02:24:06,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 in 171 msec 2024-11-08T02:24:06,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-08T02:24:06,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, REOPEN/MOVE in 490 msec 2024-11-08T02:24:06,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-08T02:24:06,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-08T02:24:06,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 911 msec 2024-11-08T02:24:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-08T02:24:06,231 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:06,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:06,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:06,233 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@591c54a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:06,234 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af92b30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:06,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:06,236 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2730ac96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:06,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:06,237 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:06,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85bc67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:06,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:06,239 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4256eebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:06,240 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e592a83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:06,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:06,241 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f620cf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:06,242 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:06,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2710d17a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,243 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:06,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:06,244 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf82dd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:06,245 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48f3248e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:06,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:06,246 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2d4c4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:06,248 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b5abb5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:06,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:06,249 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d353d5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:06,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:06,250 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:06,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66dcdc40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:06,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:06,252 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3d54c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:06,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:06,253 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:06,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2887ea6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:06,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:06,255 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:06,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:06,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:06,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@762f1269, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:06,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:06,256 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132405d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:06,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:06,258 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7faebbe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:06,260 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ad381de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:06,261 DEBUG [Thread-1535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:06,261 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:06,261 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:06,261 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:06,261 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:06,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:06,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:06,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:06,261 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:06,262 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:06,262 DEBUG [Thread-1533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:06,262 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:06,263 DEBUG [Thread-1529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:06,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:06,264 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:06,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:06,264 DEBUG [Thread-1531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:06,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:06,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:06,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:06,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:06,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:06,265 DEBUG [Thread-1531 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:06,265 DEBUG [Thread-1531 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:06,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:06,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:06,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-08T02:24:06,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:06,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:06,267 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T02:24:06,267 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:24:06,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:06,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:24:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:24:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:24:06,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:24:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:24:06,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:24:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:24:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:24:06,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:24:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,280 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:06,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:24:06,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108332259eaa51b490c93568b3c0ddb3b07_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032646277/Put/seqid=0 2024-11-08T02:24:06,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742160_1336 (size=12154) 2024-11-08T02:24:06,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032706357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032706358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032706358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032706367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032706368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T02:24:06,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T02:24:06,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:06,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:06,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:06,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032706478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032706478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032706478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032706478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032706488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T02:24:06,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T02:24:06,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:06,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:06,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:06,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032706688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032706688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032706689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032706689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032706698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:06,731 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:06,735 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108332259eaa51b490c93568b3c0ddb3b07_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108332259eaa51b490c93568b3c0ddb3b07_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:06,740 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/304946d655a345108723bf54ed202abf, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:06,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/304946d655a345108723bf54ed202abf is 175, key is test_row_0/A:col10/1731032646277/Put/seqid=0 2024-11-08T02:24:06,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742161_1337 (size=30955) 2024-11-08T02:24:06,748 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/304946d655a345108723bf54ed202abf 2024-11-08T02:24:06,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/4fc00f03244243c99014fb8bc65506d2 is 50, key is test_row_0/B:col10/1731032646277/Put/seqid=0 2024-11-08T02:24:06,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742162_1338 (size=12001) 2024-11-08T02:24:06,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T02:24:06,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:06,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:06,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:06,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T02:24:06,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T02:24:06,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:06,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:06,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:06,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:06,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032706997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032706998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032706998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032706998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032707008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T02:24:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:07,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:07,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:07,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/4fc00f03244243c99014fb8bc65506d2 2024-11-08T02:24:07,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/c5587019912e4f539cd038706cfc0d36 is 50, key is test_row_0/C:col10/1731032646277/Put/seqid=0 2024-11-08T02:24:07,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742163_1339 (size=12001) 2024-11-08T02:24:07,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/c5587019912e4f539cd038706cfc0d36 2024-11-08T02:24:07,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/304946d655a345108723bf54ed202abf as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/304946d655a345108723bf54ed202abf 2024-11-08T02:24:07,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/304946d655a345108723bf54ed202abf, entries=150, sequenceid=15, filesize=30.2 K 2024-11-08T02:24:07,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/4fc00f03244243c99014fb8bc65506d2 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/4fc00f03244243c99014fb8bc65506d2 2024-11-08T02:24:07,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/4fc00f03244243c99014fb8bc65506d2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-08T02:24:07,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/c5587019912e4f539cd038706cfc0d36 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/c5587019912e4f539cd038706cfc0d36 2024-11-08T02:24:07,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/c5587019912e4f539cd038706cfc0d36, entries=150, sequenceid=15, filesize=11.7 K 2024-11-08T02:24:07,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a03465591aab9088b0f79bf5ef3a6e88 in 948ms, sequenceid=15, compaction requested=false 2024-11-08T02:24:07,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:07,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T02:24:07,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:07,256 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:24:07,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110818947377c8d44619ad8bddc4187df785_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032646366/Put/seqid=0 2024-11-08T02:24:07,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742164_1340 (size=12154) 2024-11-08T02:24:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:07,290 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110818947377c8d44619ad8bddc4187df785_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110818947377c8d44619ad8bddc4187df785_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:07,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/a826005de4ef4aeab8e32fa72e148140, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:07,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/a826005de4ef4aeab8e32fa72e148140 is 175, key is test_row_0/A:col10/1731032646366/Put/seqid=0 2024-11-08T02:24:07,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742165_1341 (size=30955) 2024-11-08T02:24:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T02:24:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:07,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:07,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032707515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032707518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032707519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032707520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032707521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032707628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032707628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032707628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032707637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,706 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/a826005de4ef4aeab8e32fa72e148140 2024-11-08T02:24:07,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/ecf72282e1504bc3b8e451fff4b35f1d is 50, key is test_row_0/B:col10/1731032646366/Put/seqid=0 2024-11-08T02:24:07,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742166_1342 (size=12001) 2024-11-08T02:24:07,720 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/ecf72282e1504bc3b8e451fff4b35f1d 2024-11-08T02:24:07,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f4c62a50b2334b578f6b4617fae1f7e8 is 50, key is test_row_0/C:col10/1731032646366/Put/seqid=0 2024-11-08T02:24:07,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742167_1343 (size=12001) 2024-11-08T02:24:07,769 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f4c62a50b2334b578f6b4617fae1f7e8 2024-11-08T02:24:07,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/a826005de4ef4aeab8e32fa72e148140 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/a826005de4ef4aeab8e32fa72e148140 2024-11-08T02:24:07,787 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/a826005de4ef4aeab8e32fa72e148140, entries=150, sequenceid=40, filesize=30.2 K 2024-11-08T02:24:07,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/ecf72282e1504bc3b8e451fff4b35f1d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/ecf72282e1504bc3b8e451fff4b35f1d 2024-11-08T02:24:07,805 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/ecf72282e1504bc3b8e451fff4b35f1d, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:24:07,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f4c62a50b2334b578f6b4617fae1f7e8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f4c62a50b2334b578f6b4617fae1f7e8 2024-11-08T02:24:07,809 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f4c62a50b2334b578f6b4617fae1f7e8, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:24:07,810 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a03465591aab9088b0f79bf5ef3a6e88 in 554ms, sequenceid=40, compaction requested=false 2024-11-08T02:24:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-08T02:24:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-08T02:24:07,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-08T02:24:07,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5460 sec 2024-11-08T02:24:07,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.5500 sec 2024-11-08T02:24:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:07,841 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:07,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:24:07,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108758ec9fabd8245859008a5bead4fccad_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032647511/Put/seqid=0 2024-11-08T02:24:07,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742168_1344 (size=14594) 2024-11-08T02:24:07,890 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:24:07,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032707884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032707885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032707886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032707896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:07,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032707998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032708008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032708008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032708008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032708218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032708218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032708228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032708228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,260 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:08,263 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108758ec9fabd8245859008a5bead4fccad_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108758ec9fabd8245859008a5bead4fccad_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:08,267 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/e3fe1ad648c3443a8668226af7dd67d3, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:08,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/e3fe1ad648c3443a8668226af7dd67d3 is 175, key is test_row_0/A:col10/1731032647511/Put/seqid=0 2024-11-08T02:24:08,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742169_1345 (size=39549) 2024-11-08T02:24:08,277 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/e3fe1ad648c3443a8668226af7dd67d3 2024-11-08T02:24:08,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/3f19595d92a84c8b9279706a819b98f3 is 50, key is test_row_0/B:col10/1731032647511/Put/seqid=0 2024-11-08T02:24:08,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742170_1346 (size=12001) 2024-11-08T02:24:08,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/3f19595d92a84c8b9279706a819b98f3 2024-11-08T02:24:08,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/8de7af73433a422091ecd87992f3ab58 is 50, key is test_row_0/C:col10/1731032647511/Put/seqid=0 2024-11-08T02:24:08,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742171_1347 (size=12001) 2024-11-08T02:24:08,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/8de7af73433a422091ecd87992f3ab58 2024-11-08T02:24:08,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/e3fe1ad648c3443a8668226af7dd67d3 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/e3fe1ad648c3443a8668226af7dd67d3 2024-11-08T02:24:08,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/e3fe1ad648c3443a8668226af7dd67d3, entries=200, sequenceid=52, filesize=38.6 K 2024-11-08T02:24:08,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/3f19595d92a84c8b9279706a819b98f3 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/3f19595d92a84c8b9279706a819b98f3 2024-11-08T02:24:08,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/3f19595d92a84c8b9279706a819b98f3, entries=150, sequenceid=52, filesize=11.7 K 2024-11-08T02:24:08,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/8de7af73433a422091ecd87992f3ab58 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/8de7af73433a422091ecd87992f3ab58 2024-11-08T02:24:08,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/8de7af73433a422091ecd87992f3ab58, entries=150, sequenceid=52, filesize=11.7 K 2024-11-08T02:24:08,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a03465591aab9088b0f79bf5ef3a6e88 in 495ms, sequenceid=52, compaction requested=true 2024-11-08T02:24:08,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:08,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:08,336 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:08,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:08,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:08,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:08,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:08,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T02:24:08,336 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:08,338 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:08,338 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/A is initiating minor compaction (all files) 2024-11-08T02:24:08,338 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/A in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:08,338 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/304946d655a345108723bf54ed202abf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/a826005de4ef4aeab8e32fa72e148140, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/e3fe1ad648c3443a8668226af7dd67d3] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=99.1 K 2024-11-08T02:24:08,338 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:08,338 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/304946d655a345108723bf54ed202abf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/a826005de4ef4aeab8e32fa72e148140, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/e3fe1ad648c3443a8668226af7dd67d3] 2024-11-08T02:24:08,339 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:08,339 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 304946d655a345108723bf54ed202abf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731032646276 2024-11-08T02:24:08,339 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/B is initiating minor compaction (all files) 2024-11-08T02:24:08,339 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/B in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:08,339 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/4fc00f03244243c99014fb8bc65506d2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/ecf72282e1504bc3b8e451fff4b35f1d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/3f19595d92a84c8b9279706a819b98f3] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=35.2 K 2024-11-08T02:24:08,339 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting a826005de4ef4aeab8e32fa72e148140, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731032646291 2024-11-08T02:24:08,339 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fc00f03244243c99014fb8bc65506d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731032646276 2024-11-08T02:24:08,339 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3fe1ad648c3443a8668226af7dd67d3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731032647510 2024-11-08T02:24:08,339 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting ecf72282e1504bc3b8e451fff4b35f1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731032646291 2024-11-08T02:24:08,340 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f19595d92a84c8b9279706a819b98f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731032647510 2024-11-08T02:24:08,346 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:08,349 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#B#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:08,350 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/332b57db6ddc4f9788584b5dfa73acda is 50, key is test_row_0/B:col10/1731032647511/Put/seqid=0 2024-11-08T02:24:08,353 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108fbc05d7df57c4d04a962dc70c7ac79b2_a03465591aab9088b0f79bf5ef3a6e88 store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:08,355 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108fbc05d7df57c4d04a962dc70c7ac79b2_a03465591aab9088b0f79bf5ef3a6e88, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:08,355 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fbc05d7df57c4d04a962dc70c7ac79b2_a03465591aab9088b0f79bf5ef3a6e88 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:08,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742172_1348 (size=12104) 2024-11-08T02:24:08,371 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/332b57db6ddc4f9788584b5dfa73acda as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/332b57db6ddc4f9788584b5dfa73acda 2024-11-08T02:24:08,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742173_1349 (size=4469) 2024-11-08T02:24:08,377 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#A#compaction#300 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:08,377 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/ad66fda3508349f084863cdf86b77952 is 175, key is test_row_0/A:col10/1731032647511/Put/seqid=0 2024-11-08T02:24:08,383 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/B of a03465591aab9088b0f79bf5ef3a6e88 into 332b57db6ddc4f9788584b5dfa73acda(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:08,384 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:08,384 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/B, priority=13, startTime=1731032648336; duration=0sec 2024-11-08T02:24:08,384 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:08,384 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:B 2024-11-08T02:24:08,384 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:08,392 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:08,393 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/C is initiating minor compaction (all files) 2024-11-08T02:24:08,393 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/C in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:08,393 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/c5587019912e4f539cd038706cfc0d36, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f4c62a50b2334b578f6b4617fae1f7e8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/8de7af73433a422091ecd87992f3ab58] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=35.2 K 2024-11-08T02:24:08,395 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c5587019912e4f539cd038706cfc0d36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731032646276 2024-11-08T02:24:08,395 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting f4c62a50b2334b578f6b4617fae1f7e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731032646291 2024-11-08T02:24:08,396 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 8de7af73433a422091ecd87992f3ab58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731032647510 2024-11-08T02:24:08,409 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:08,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742174_1350 (size=31058) 2024-11-08T02:24:08,409 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/cc8a91776e744bb7a73e9d1f27f7233d is 50, key is test_row_0/C:col10/1731032647511/Put/seqid=0 2024-11-08T02:24:08,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742175_1351 (size=12104) 2024-11-08T02:24:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T02:24:08,418 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:08,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-08T02:24:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T02:24:08,419 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:08,420 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:08,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T02:24:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:08,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:24:08,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110817edb8a003124a368588c9574c6ebf36_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032647884/Put/seqid=0 2024-11-08T02:24:08,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742176_1352 (size=14594) 2024-11-08T02:24:08,551 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:08,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110817edb8a003124a368588c9574c6ebf36_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110817edb8a003124a368588c9574c6ebf36_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:08,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032708546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032708547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,556 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/554333007a564f669ad42adc7b755acf, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/554333007a564f669ad42adc7b755acf is 175, key is test_row_0/A:col10/1731032647884/Put/seqid=0 2024-11-08T02:24:08,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032708547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032708549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032708550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742177_1353 (size=39549) 2024-11-08T02:24:08,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T02:24:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:08,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:08,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032708658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032708658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032708668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032708668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032708668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T02:24:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:08,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T02:24:08,813 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/ad66fda3508349f084863cdf86b77952 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/ad66fda3508349f084863cdf86b77952 2024-11-08T02:24:08,818 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/A of a03465591aab9088b0f79bf5ef3a6e88 into ad66fda3508349f084863cdf86b77952(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:08,818 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:08,818 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/A, priority=13, startTime=1731032648336; duration=0sec 2024-11-08T02:24:08,818 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:08,818 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:A 2024-11-08T02:24:08,818 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/cc8a91776e744bb7a73e9d1f27f7233d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/cc8a91776e744bb7a73e9d1f27f7233d 2024-11-08T02:24:08,823 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/C of a03465591aab9088b0f79bf5ef3a6e88 into cc8a91776e744bb7a73e9d1f27f7233d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:08,823 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:08,823 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/C, priority=13, startTime=1731032648336; duration=0sec 2024-11-08T02:24:08,823 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:08,823 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:C 2024-11-08T02:24:08,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032708868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032708868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T02:24:08,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:08,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:08,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:08,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032708877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032708878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032708878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:08,966 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/554333007a564f669ad42adc7b755acf 2024-11-08T02:24:08,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/df0591f03b2e4e36aa6eacda2419f219 is 50, key is test_row_0/B:col10/1731032647884/Put/seqid=0 2024-11-08T02:24:08,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742178_1354 (size=12001) 2024-11-08T02:24:08,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/df0591f03b2e4e36aa6eacda2419f219 2024-11-08T02:24:08,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/2e1272ab6dd548f8b4b7e69033d0d0cf is 50, key is test_row_0/C:col10/1731032647884/Put/seqid=0 2024-11-08T02:24:09,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742179_1355 (size=12001) 2024-11-08T02:24:09,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T02:24:09,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:09,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:09,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:09,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:09,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T02:24:09,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032709178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T02:24:09,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:09,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:09,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:09,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:09,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:09,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032709188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032709187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032709188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032709188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T02:24:09,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:09,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:09,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:09,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:09,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:09,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/2e1272ab6dd548f8b4b7e69033d0d0cf 2024-11-08T02:24:09,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/554333007a564f669ad42adc7b755acf as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/554333007a564f669ad42adc7b755acf 2024-11-08T02:24:09,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/554333007a564f669ad42adc7b755acf, entries=200, sequenceid=78, filesize=38.6 K 2024-11-08T02:24:09,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/df0591f03b2e4e36aa6eacda2419f219 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/df0591f03b2e4e36aa6eacda2419f219 2024-11-08T02:24:09,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/df0591f03b2e4e36aa6eacda2419f219, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T02:24:09,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/2e1272ab6dd548f8b4b7e69033d0d0cf as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/2e1272ab6dd548f8b4b7e69033d0d0cf 2024-11-08T02:24:09,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/2e1272ab6dd548f8b4b7e69033d0d0cf, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T02:24:09,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a03465591aab9088b0f79bf5ef3a6e88 in 898ms, sequenceid=78, compaction requested=false 2024-11-08T02:24:09,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:09,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T02:24:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:09,489 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:24:09,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110842888b2ae003496ba3be501010dd020e_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032648545/Put/seqid=0 2024-11-08T02:24:09,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742180_1356 (size=12154) 2024-11-08T02:24:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T02:24:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:09,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:09,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032709729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032709729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032709730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032709733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032709737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032709847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032709847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032709847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032709848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032709848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:09,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:09,910 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110842888b2ae003496ba3be501010dd020e_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110842888b2ae003496ba3be501010dd020e_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:09,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/c41a068134204d10a982b4237c372601, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:09,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/c41a068134204d10a982b4237c372601 is 175, key is test_row_0/A:col10/1731032648545/Put/seqid=0 2024-11-08T02:24:09,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742181_1357 (size=30955) 2024-11-08T02:24:10,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032710068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032710068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032710068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032710068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032710068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,318 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/c41a068134204d10a982b4237c372601 2024-11-08T02:24:10,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/328afc932d4645cbb6300cb58cc30793 is 50, key is test_row_0/B:col10/1731032648545/Put/seqid=0 2024-11-08T02:24:10,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742182_1358 (size=12001) 2024-11-08T02:24:10,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032710388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032710388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032710388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032710389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032710389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T02:24:10,727 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/328afc932d4645cbb6300cb58cc30793 2024-11-08T02:24:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/ff6e1142148945f5b2b7a98de11c8dc0 is 50, key is test_row_0/C:col10/1731032648545/Put/seqid=0 2024-11-08T02:24:10,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742183_1359 (size=12001) 2024-11-08T02:24:10,737 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/ff6e1142148945f5b2b7a98de11c8dc0 2024-11-08T02:24:10,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/c41a068134204d10a982b4237c372601 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/c41a068134204d10a982b4237c372601 2024-11-08T02:24:10,744 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/c41a068134204d10a982b4237c372601, entries=150, sequenceid=91, filesize=30.2 K 2024-11-08T02:24:10,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/328afc932d4645cbb6300cb58cc30793 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/328afc932d4645cbb6300cb58cc30793 2024-11-08T02:24:10,748 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/328afc932d4645cbb6300cb58cc30793, entries=150, sequenceid=91, filesize=11.7 K 2024-11-08T02:24:10,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/ff6e1142148945f5b2b7a98de11c8dc0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/ff6e1142148945f5b2b7a98de11c8dc0 2024-11-08T02:24:10,756 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/ff6e1142148945f5b2b7a98de11c8dc0, entries=150, sequenceid=91, filesize=11.7 K 2024-11-08T02:24:10,756 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a03465591aab9088b0f79bf5ef3a6e88 in 1267ms, sequenceid=91, compaction requested=true 2024-11-08T02:24:10,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:10,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-08T02:24:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-08T02:24:10,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-08T02:24:10,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3370 sec 2024-11-08T02:24:10,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3410 sec 2024-11-08T02:24:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:10,914 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:10,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:24:10,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032710918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032710919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032710920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032710927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108bffd68d0d8ea4d52a91de48fcd681fac_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032649727/Put/seqid=0 2024-11-08T02:24:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032710928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:10,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742184_1360 (size=14594) 2024-11-08T02:24:10,942 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:10,946 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108bffd68d0d8ea4d52a91de48fcd681fac_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bffd68d0d8ea4d52a91de48fcd681fac_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:10,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5f1e7915f01f4b379eafda9c4f1ac8ca, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:10,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5f1e7915f01f4b379eafda9c4f1ac8ca is 175, key is test_row_0/A:col10/1731032649727/Put/seqid=0 2024-11-08T02:24:10,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742185_1361 (size=39549) 2024-11-08T02:24:11,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032711038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032711038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032711038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032711038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032711048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032711248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032711248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032711249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032711258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032711258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,358 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5f1e7915f01f4b379eafda9c4f1ac8ca 2024-11-08T02:24:11,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/bb3bd0b7b2e04aa0a8c043961cd922a8 is 50, key is test_row_0/B:col10/1731032649727/Put/seqid=0 2024-11-08T02:24:11,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742186_1362 (size=12001) 2024-11-08T02:24:11,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032711558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032711558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032711558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032711577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032711577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:11,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:11,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/bb3bd0b7b2e04aa0a8c043961cd922a8 2024-11-08T02:24:11,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/93075fe379124dffb321b6fa99e162af is 50, key is test_row_0/C:col10/1731032649727/Put/seqid=0 2024-11-08T02:24:11,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742187_1363 (size=12001) 2024-11-08T02:24:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032712078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:12,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032712078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032712078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:12,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032712098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:12,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032712098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:12,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/93075fe379124dffb321b6fa99e162af 2024-11-08T02:24:12,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5f1e7915f01f4b379eafda9c4f1ac8ca as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5f1e7915f01f4b379eafda9c4f1ac8ca 2024-11-08T02:24:12,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5f1e7915f01f4b379eafda9c4f1ac8ca, entries=200, sequenceid=117, filesize=38.6 K 2024-11-08T02:24:12,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/bb3bd0b7b2e04aa0a8c043961cd922a8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/bb3bd0b7b2e04aa0a8c043961cd922a8 2024-11-08T02:24:12,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/bb3bd0b7b2e04aa0a8c043961cd922a8, entries=150, sequenceid=117, filesize=11.7 K 2024-11-08T02:24:12,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/93075fe379124dffb321b6fa99e162af as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/93075fe379124dffb321b6fa99e162af 2024-11-08T02:24:12,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/93075fe379124dffb321b6fa99e162af, entries=150, sequenceid=117, filesize=11.7 K 2024-11-08T02:24:12,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a03465591aab9088b0f79bf5ef3a6e88 in 1303ms, sequenceid=117, compaction requested=true 2024-11-08T02:24:12,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:12,217 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:24:12,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:12,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:12,218 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:24:12,219 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:24:12,219 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/A is initiating minor compaction (all files) 2024-11-08T02:24:12,219 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/A in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:12,219 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/ad66fda3508349f084863cdf86b77952, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/554333007a564f669ad42adc7b755acf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/c41a068134204d10a982b4237c372601, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5f1e7915f01f4b379eafda9c4f1ac8ca] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=137.8 K 2024-11-08T02:24:12,219 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:12,219 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/ad66fda3508349f084863cdf86b77952, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/554333007a564f669ad42adc7b755acf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/c41a068134204d10a982b4237c372601, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5f1e7915f01f4b379eafda9c4f1ac8ca] 2024-11-08T02:24:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:12,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:12,220 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad66fda3508349f084863cdf86b77952, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731032647510 2024-11-08T02:24:12,220 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 554333007a564f669ad42adc7b755acf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731032647883 2024-11-08T02:24:12,220 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:24:12,220 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/B is initiating minor compaction (all files) 2024-11-08T02:24:12,220 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/B in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:12,220 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/332b57db6ddc4f9788584b5dfa73acda, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/df0591f03b2e4e36aa6eacda2419f219, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/328afc932d4645cbb6300cb58cc30793, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/bb3bd0b7b2e04aa0a8c043961cd922a8] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=47.0 K 2024-11-08T02:24:12,220 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c41a068134204d10a982b4237c372601, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731032648545 2024-11-08T02:24:12,220 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 332b57db6ddc4f9788584b5dfa73acda, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731032647510 2024-11-08T02:24:12,221 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f1e7915f01f4b379eafda9c4f1ac8ca, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731032649727 2024-11-08T02:24:12,221 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting df0591f03b2e4e36aa6eacda2419f219, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731032647883 2024-11-08T02:24:12,221 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 328afc932d4645cbb6300cb58cc30793, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731032648545 2024-11-08T02:24:12,222 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting bb3bd0b7b2e04aa0a8c043961cd922a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731032649727 2024-11-08T02:24:12,226 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:12,237 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411084a3bb876d4a54be6af7e5103e8ff5931_a03465591aab9088b0f79bf5ef3a6e88 store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:12,239 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411084a3bb876d4a54be6af7e5103e8ff5931_a03465591aab9088b0f79bf5ef3a6e88, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:12,239 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084a3bb876d4a54be6af7e5103e8ff5931_a03465591aab9088b0f79bf5ef3a6e88 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:12,242 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#B#compaction#313 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:12,243 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/f47d00c6579740a98ba6c9ca14d7020e is 50, key is test_row_0/B:col10/1731032649727/Put/seqid=0 2024-11-08T02:24:12,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742188_1364 (size=4469) 2024-11-08T02:24:12,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742189_1365 (size=12241) 2024-11-08T02:24:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T02:24:12,578 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:12,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-08T02:24:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T02:24:12,580 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:12,581 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:12,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:12,655 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#A#compaction#312 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:12,656 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/b642c3d133f64d9ca0bc66e0d33bb740 is 175, key is test_row_0/A:col10/1731032649727/Put/seqid=0 2024-11-08T02:24:12,667 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/f47d00c6579740a98ba6c9ca14d7020e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/f47d00c6579740a98ba6c9ca14d7020e 2024-11-08T02:24:12,671 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/B of a03465591aab9088b0f79bf5ef3a6e88 into f47d00c6579740a98ba6c9ca14d7020e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:12,671 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:12,671 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/B, priority=12, startTime=1731032652218; duration=0sec 2024-11-08T02:24:12,672 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:12,672 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:B 2024-11-08T02:24:12,672 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:24:12,673 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:24:12,673 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/C is initiating minor compaction (all files) 2024-11-08T02:24:12,673 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/C in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:12,673 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/cc8a91776e744bb7a73e9d1f27f7233d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/2e1272ab6dd548f8b4b7e69033d0d0cf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/ff6e1142148945f5b2b7a98de11c8dc0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/93075fe379124dffb321b6fa99e162af] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=47.0 K 2024-11-08T02:24:12,674 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting cc8a91776e744bb7a73e9d1f27f7233d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731032647510 2024-11-08T02:24:12,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742190_1366 (size=31195) 2024-11-08T02:24:12,676 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e1272ab6dd548f8b4b7e69033d0d0cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731032647883 2024-11-08T02:24:12,680 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting ff6e1142148945f5b2b7a98de11c8dc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731032648545 2024-11-08T02:24:12,680 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 93075fe379124dffb321b6fa99e162af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731032649727 2024-11-08T02:24:12,686 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/b642c3d133f64d9ca0bc66e0d33bb740 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b642c3d133f64d9ca0bc66e0d33bb740 2024-11-08T02:24:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T02:24:12,693 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/A of a03465591aab9088b0f79bf5ef3a6e88 into b642c3d133f64d9ca0bc66e0d33bb740(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:12,693 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:12,693 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/A, priority=12, startTime=1731032652217; duration=0sec 2024-11-08T02:24:12,693 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:12,693 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:A 2024-11-08T02:24:12,698 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#C#compaction#314 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:12,698 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/6025b371c9dd4a3aad743f4891ee1bbe is 50, key is test_row_0/C:col10/1731032649727/Put/seqid=0 2024-11-08T02:24:12,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742191_1367 (size=12241) 2024-11-08T02:24:12,710 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/6025b371c9dd4a3aad743f4891ee1bbe as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6025b371c9dd4a3aad743f4891ee1bbe 2024-11-08T02:24:12,714 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/C of a03465591aab9088b0f79bf5ef3a6e88 into 6025b371c9dd4a3aad743f4891ee1bbe(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:12,714 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:12,714 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/C, priority=12, startTime=1731032652219; duration=0sec 2024-11-08T02:24:12,714 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:12,714 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:C 2024-11-08T02:24:12,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T02:24:12,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:12,733 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T02:24:12,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c70eb3b1fd2b4ddd884814ede999923d_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032650918/Put/seqid=0 2024-11-08T02:24:12,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742192_1368 (size=12154) 2024-11-08T02:24:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T02:24:13,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:13,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:13,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:13,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032713133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032713134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032713138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,146 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c70eb3b1fd2b4ddd884814ede999923d_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c70eb3b1fd2b4ddd884814ede999923d_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:13,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/1a5a3a8317a245cbbb6fd727f115678e, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:13,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/1a5a3a8317a245cbbb6fd727f115678e is 175, key is test_row_0/A:col10/1731032650918/Put/seqid=0 2024-11-08T02:24:13,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032713139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032713143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742193_1369 (size=30955) 2024-11-08T02:24:13,154 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/1a5a3a8317a245cbbb6fd727f115678e 2024-11-08T02:24:13,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/6eb9fece42394040b9d57b4392bbe44d is 50, key is test_row_0/B:col10/1731032650918/Put/seqid=0 2024-11-08T02:24:13,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742194_1370 (size=12001) 2024-11-08T02:24:13,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T02:24:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032713248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032713248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032713248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032713257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032713257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032713468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032713468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032713468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032713468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032713478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,568 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/6eb9fece42394040b9d57b4392bbe44d 2024-11-08T02:24:13,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/1623cf7d12164e1e9dd00c5de98d3db1 is 50, key is test_row_0/C:col10/1731032650918/Put/seqid=0 2024-11-08T02:24:13,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742195_1371 (size=12001) 2024-11-08T02:24:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T02:24:13,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032713778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032713778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032713778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032713778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032713788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:13,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:13,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:13,988 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/1623cf7d12164e1e9dd00c5de98d3db1 2024-11-08T02:24:13,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/1a5a3a8317a245cbbb6fd727f115678e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/1a5a3a8317a245cbbb6fd727f115678e 2024-11-08T02:24:13,996 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/1a5a3a8317a245cbbb6fd727f115678e, entries=150, sequenceid=130, filesize=30.2 K 2024-11-08T02:24:13,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/6eb9fece42394040b9d57b4392bbe44d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6eb9fece42394040b9d57b4392bbe44d 2024-11-08T02:24:14,000 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6eb9fece42394040b9d57b4392bbe44d, entries=150, sequenceid=130, filesize=11.7 K 2024-11-08T02:24:14,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/1623cf7d12164e1e9dd00c5de98d3db1 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/1623cf7d12164e1e9dd00c5de98d3db1 2024-11-08T02:24:14,005 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/1623cf7d12164e1e9dd00c5de98d3db1, entries=150, sequenceid=130, filesize=11.7 K 2024-11-08T02:24:14,006 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for a03465591aab9088b0f79bf5ef3a6e88 in 1273ms, sequenceid=130, compaction requested=false 2024-11-08T02:24:14,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:14,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:14,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-08T02:24:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-08T02:24:14,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-08T02:24:14,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4270 sec 2024-11-08T02:24:14,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4320 sec 2024-11-08T02:24:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:14,295 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:14,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T02:24:14,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ac37aaf482cf43dfbc75b12cab952788_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032653133/Put/seqid=0 2024-11-08T02:24:14,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742196_1372 (size=14794) 2024-11-08T02:24:14,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032714299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032714308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032714309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032714310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032714310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032714418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032714418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032714418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032714418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032714628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032714628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032714628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032714628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,707 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:14,710 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ac37aaf482cf43dfbc75b12cab952788_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ac37aaf482cf43dfbc75b12cab952788_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:14,711 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/fc54fcf1e27244d3982fa90c64443ec8, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:14,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/fc54fcf1e27244d3982fa90c64443ec8 is 175, key is test_row_0/A:col10/1731032653133/Put/seqid=0 2024-11-08T02:24:14,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742197_1373 (size=39749) 2024-11-08T02:24:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T02:24:14,728 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:14,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-08T02:24:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T02:24:14,731 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:14,731 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:14,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T02:24:14,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-08T02:24:14,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:14,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:14,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:14,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:14,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:14,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032714938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032714948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032714948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032714948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:15,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-08T02:24:15,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:15,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T02:24:15,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/fc54fcf1e27244d3982fa90c64443ec8 2024-11-08T02:24:15,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/906f5f93102648ed993e29ff374fa779 is 50, key is test_row_0/B:col10/1731032653133/Put/seqid=0 2024-11-08T02:24:15,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742198_1374 (size=12151) 2024-11-08T02:24:15,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-08T02:24:15,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:15,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032715318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:15,325 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in a03465591aab9088b0f79bf5ef3a6e88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:15,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:15,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-08T02:24:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T02:24:15,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032715457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:15,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032715457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:15,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032715458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:15,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:15,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032715468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:15,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-08T02:24:15,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:15,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/906f5f93102648ed993e29ff374fa779 2024-11-08T02:24:15,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/71ce85087b3f43818a1925d383a6e367 is 50, key is test_row_0/C:col10/1731032653133/Put/seqid=0 2024-11-08T02:24:15,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742199_1375 (size=12151) 2024-11-08T02:24:15,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-08T02:24:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-08T02:24:15,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:15,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T02:24:15,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/71ce85087b3f43818a1925d383a6e367 2024-11-08T02:24:15,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/fc54fcf1e27244d3982fa90c64443ec8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/fc54fcf1e27244d3982fa90c64443ec8 2024-11-08T02:24:15,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/fc54fcf1e27244d3982fa90c64443ec8, entries=200, sequenceid=157, filesize=38.8 K 2024-11-08T02:24:15,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/906f5f93102648ed993e29ff374fa779 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/906f5f93102648ed993e29ff374fa779 2024-11-08T02:24:15,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-08T02:24:15,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:15,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:15,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/906f5f93102648ed993e29ff374fa779, entries=150, sequenceid=157, filesize=11.9 K 2024-11-08T02:24:15,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/71ce85087b3f43818a1925d383a6e367 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/71ce85087b3f43818a1925d383a6e367 2024-11-08T02:24:15,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/71ce85087b3f43818a1925d383a6e367, entries=150, sequenceid=157, filesize=11.9 K 2024-11-08T02:24:15,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a03465591aab9088b0f79bf5ef3a6e88 in 1666ms, sequenceid=157, compaction requested=true 2024-11-08T02:24:15,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:15,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:15,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:15,962 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:15,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:15,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:15,962 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:15,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:15,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:15,962 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:15,962 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:15,962 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/B is initiating minor compaction (all files) 2024-11-08T02:24:15,962 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/A is initiating minor compaction (all files) 2024-11-08T02:24:15,963 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/A in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,963 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/B in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,963 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b642c3d133f64d9ca0bc66e0d33bb740, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/1a5a3a8317a245cbbb6fd727f115678e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/fc54fcf1e27244d3982fa90c64443ec8] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=99.5 K 2024-11-08T02:24:15,963 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/f47d00c6579740a98ba6c9ca14d7020e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6eb9fece42394040b9d57b4392bbe44d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/906f5f93102648ed993e29ff374fa779] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=35.5 K 2024-11-08T02:24:15,963 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:15,963 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b642c3d133f64d9ca0bc66e0d33bb740, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/1a5a3a8317a245cbbb6fd727f115678e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/fc54fcf1e27244d3982fa90c64443ec8] 2024-11-08T02:24:15,963 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b642c3d133f64d9ca0bc66e0d33bb740, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731032649727 2024-11-08T02:24:15,963 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting f47d00c6579740a98ba6c9ca14d7020e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731032649727 2024-11-08T02:24:15,963 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb9fece42394040b9d57b4392bbe44d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731032650918 2024-11-08T02:24:15,963 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a5a3a8317a245cbbb6fd727f115678e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731032650918 2024-11-08T02:24:15,964 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc54fcf1e27244d3982fa90c64443ec8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731032653133 2024-11-08T02:24:15,964 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 906f5f93102648ed993e29ff374fa779, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731032653133 2024-11-08T02:24:15,975 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:15,986 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#B#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:15,987 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/85976ccb556048919da31321cdbd5fcf is 50, key is test_row_0/B:col10/1731032653133/Put/seqid=0 2024-11-08T02:24:15,988 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110814b05737a172424d90d9585e981d463d_a03465591aab9088b0f79bf5ef3a6e88 store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:15,995 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110814b05737a172424d90d9585e981d463d_a03465591aab9088b0f79bf5ef3a6e88, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:15,995 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110814b05737a172424d90d9585e981d463d_a03465591aab9088b0f79bf5ef3a6e88 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:16,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742200_1376 (size=12493) 2024-11-08T02:24:16,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742201_1377 (size=4469) 2024-11-08T02:24:16,012 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/85976ccb556048919da31321cdbd5fcf as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/85976ccb556048919da31321cdbd5fcf 2024-11-08T02:24:16,013 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#A#compaction#321 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:16,013 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/dda74695d723437fbb7c819c6f47f6e0 is 175, key is test_row_0/A:col10/1731032653133/Put/seqid=0 2024-11-08T02:24:16,022 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/B of a03465591aab9088b0f79bf5ef3a6e88 into 85976ccb556048919da31321cdbd5fcf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:16,022 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:16,022 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/B, priority=13, startTime=1731032655962; duration=0sec 2024-11-08T02:24:16,022 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:16,023 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:B 2024-11-08T02:24:16,023 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:16,025 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:16,025 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/C is initiating minor compaction (all files) 2024-11-08T02:24:16,025 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/C in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:16,025 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6025b371c9dd4a3aad743f4891ee1bbe, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/1623cf7d12164e1e9dd00c5de98d3db1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/71ce85087b3f43818a1925d383a6e367] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=35.5 K 2024-11-08T02:24:16,026 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 6025b371c9dd4a3aad743f4891ee1bbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731032649727 2024-11-08T02:24:16,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742202_1378 (size=31447) 2024-11-08T02:24:16,026 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 1623cf7d12164e1e9dd00c5de98d3db1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731032650918 2024-11-08T02:24:16,026 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ce85087b3f43818a1925d383a6e367, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731032653133 2024-11-08T02:24:16,034 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/dda74695d723437fbb7c819c6f47f6e0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/dda74695d723437fbb7c819c6f47f6e0 2024-11-08T02:24:16,037 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#C#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:16,038 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/c4f1a5fa410f401e919215179ac26dd0 is 50, key is test_row_0/C:col10/1731032653133/Put/seqid=0 2024-11-08T02:24:16,041 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/A of a03465591aab9088b0f79bf5ef3a6e88 into dda74695d723437fbb7c819c6f47f6e0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:16,041 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:16,041 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/A, priority=13, startTime=1731032655961; duration=0sec 2024-11-08T02:24:16,041 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:16,041 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:A 2024-11-08T02:24:16,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742203_1379 (size=12493) 2024-11-08T02:24:16,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-08T02:24:16,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:16,107 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-08T02:24:16,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110823cd7e12e2864534a99125be2a94fdfc_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032654309/Put/seqid=0 2024-11-08T02:24:16,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742204_1380 (size=12304) 2024-11-08T02:24:16,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:16,494 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/c4f1a5fa410f401e919215179ac26dd0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/c4f1a5fa410f401e919215179ac26dd0 2024-11-08T02:24:16,499 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/C of a03465591aab9088b0f79bf5ef3a6e88 into c4f1a5fa410f401e919215179ac26dd0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:16,499 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:16,499 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/C, priority=13, startTime=1731032655962; duration=0sec 2024-11-08T02:24:16,499 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:16,499 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:C 2024-11-08T02:24:16,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:16,531 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110823cd7e12e2864534a99125be2a94fdfc_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110823cd7e12e2864534a99125be2a94fdfc_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:16,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/3e5e226bca8e488da386874f36199ac0, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:16,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/3e5e226bca8e488da386874f36199ac0 is 175, key is test_row_0/A:col10/1731032654309/Put/seqid=0 2024-11-08T02:24:16,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032716530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032716531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742205_1381 (size=31105) 2024-11-08T02:24:16,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032716533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032716535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032716648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032716648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032716648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032716648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032716857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032716858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032716868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032716868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T02:24:16,943 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/3e5e226bca8e488da386874f36199ac0 2024-11-08T02:24:16,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/5184edd8fe274d8e97888743ca501497 is 50, key is test_row_0/B:col10/1731032654309/Put/seqid=0 2024-11-08T02:24:16,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742206_1382 (size=12151) 2024-11-08T02:24:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032717167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032717167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032717188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032717188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032717348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,355 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in a03465591aab9088b0f79bf5ef3a6e88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,358 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/5184edd8fe274d8e97888743ca501497 2024-11-08T02:24:17,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/5e81adad23004a848f93a486df98cf95 is 50, key is test_row_0/C:col10/1731032654309/Put/seqid=0 2024-11-08T02:24:17,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742207_1383 (size=12151) 2024-11-08T02:24:17,375 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/5e81adad23004a848f93a486df98cf95 2024-11-08T02:24:17,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/3e5e226bca8e488da386874f36199ac0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/3e5e226bca8e488da386874f36199ac0 2024-11-08T02:24:17,385 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/3e5e226bca8e488da386874f36199ac0, entries=150, sequenceid=168, filesize=30.4 K 2024-11-08T02:24:17,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/5184edd8fe274d8e97888743ca501497 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/5184edd8fe274d8e97888743ca501497 2024-11-08T02:24:17,389 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/5184edd8fe274d8e97888743ca501497, entries=150, sequenceid=168, filesize=11.9 K 2024-11-08T02:24:17,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/5e81adad23004a848f93a486df98cf95 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/5e81adad23004a848f93a486df98cf95 2024-11-08T02:24:17,393 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/5e81adad23004a848f93a486df98cf95, entries=150, sequenceid=168, filesize=11.9 K 2024-11-08T02:24:17,394 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for a03465591aab9088b0f79bf5ef3a6e88 in 1287ms, sequenceid=168, compaction requested=false 2024-11-08T02:24:17,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:17,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:17,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-08T02:24:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-08T02:24:17,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-08T02:24:17,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6650 sec 2024-11-08T02:24:17,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.6720 sec 2024-11-08T02:24:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:17,692 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:17,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T02:24:17,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110850170a30abb54fd4836335030156bfac_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032657692/Put/seqid=0 2024-11-08T02:24:17,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032717700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742208_1384 (size=17284) 2024-11-08T02:24:17,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032717703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032717704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032717705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032717808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032717818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032717818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032717818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:17,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032718017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032718028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032718028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032718028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,107 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,110 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110850170a30abb54fd4836335030156bfac_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110850170a30abb54fd4836335030156bfac_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:18,111 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/947c31d4da3144f783113d195e4ca5e8, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:18,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/947c31d4da3144f783113d195e4ca5e8 is 175, key is test_row_0/A:col10/1731032657692/Put/seqid=0 2024-11-08T02:24:18,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742209_1385 (size=48389) 2024-11-08T02:24:18,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032718338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032718338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032718338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032718338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,526 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/947c31d4da3144f783113d195e4ca5e8 2024-11-08T02:24:18,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/0c0dedfb48094b64bd67a56f2a5f1855 is 50, key is test_row_0/B:col10/1731032657692/Put/seqid=0 2024-11-08T02:24:18,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742210_1386 (size=12151) 2024-11-08T02:24:18,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032718847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032718848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032718848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032718848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T02:24:18,889 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:18,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-08T02:24:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T02:24:18,891 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:18,891 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:18,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:18,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/0c0dedfb48094b64bd67a56f2a5f1855 2024-11-08T02:24:18,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/d12172bfaa354d1683baca6a5940b9d0 is 50, key is test_row_0/C:col10/1731032657692/Put/seqid=0 2024-11-08T02:24:18,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742211_1387 (size=12151) 2024-11-08T02:24:18,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/d12172bfaa354d1683baca6a5940b9d0 2024-11-08T02:24:18,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/947c31d4da3144f783113d195e4ca5e8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/947c31d4da3144f783113d195e4ca5e8 2024-11-08T02:24:18,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/947c31d4da3144f783113d195e4ca5e8, entries=250, sequenceid=198, filesize=47.3 K 2024-11-08T02:24:18,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/0c0dedfb48094b64bd67a56f2a5f1855 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/0c0dedfb48094b64bd67a56f2a5f1855 2024-11-08T02:24:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/0c0dedfb48094b64bd67a56f2a5f1855, entries=150, sequenceid=198, filesize=11.9 K 2024-11-08T02:24:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/d12172bfaa354d1683baca6a5940b9d0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/d12172bfaa354d1683baca6a5940b9d0 2024-11-08T02:24:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/d12172bfaa354d1683baca6a5940b9d0, entries=150, sequenceid=198, filesize=11.9 K 2024-11-08T02:24:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for a03465591aab9088b0f79bf5ef3a6e88 in 1287ms, sequenceid=198, compaction requested=true 2024-11-08T02:24:18,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:18,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:18,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:18,979 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:18,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:18,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:18,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:18,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T02:24:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,980 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:18,980 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/A is initiating minor compaction (all files) 2024-11-08T02:24:18,980 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/A in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,980 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/dda74695d723437fbb7c819c6f47f6e0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/3e5e226bca8e488da386874f36199ac0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/947c31d4da3144f783113d195e4ca5e8] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=108.3 K 2024-11-08T02:24:18,980 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:18,980 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/dda74695d723437fbb7c819c6f47f6e0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/3e5e226bca8e488da386874f36199ac0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/947c31d4da3144f783113d195e4ca5e8] 2024-11-08T02:24:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,980 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting dda74695d723437fbb7c819c6f47f6e0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731032653133 2024-11-08T02:24:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,981 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e5e226bca8e488da386874f36199ac0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731032654297 2024-11-08T02:24:18,981 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 947c31d4da3144f783113d195e4ca5e8, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731032656527 2024-11-08T02:24:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,985 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,985 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,986 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/B is initiating minor compaction (all files) 2024-11-08T02:24:18,986 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/B in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:18,986 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/85976ccb556048919da31321cdbd5fcf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/5184edd8fe274d8e97888743ca501497, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/0c0dedfb48094b64bd67a56f2a5f1855] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=35.9 K 2024-11-08T02:24:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,986 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85976ccb556048919da31321cdbd5fcf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731032653133 2024-11-08T02:24:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,987 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5184edd8fe274d8e97888743ca501497, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731032654297 2024-11-08T02:24:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,987 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c0dedfb48094b64bd67a56f2a5f1855, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731032656527 2024-11-08T02:24:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,993 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T02:24:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,006 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#B#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,007 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/15ad8817d38f4c01867f4fef078a74b0 is 50, key is test_row_0/B:col10/1731032657692/Put/seqid=0 2024-11-08T02:24:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,008 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108ed1c6c9b9b7849538c2e63d052ec1c01_a03465591aab9088b0f79bf5ef3a6e88 store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,010 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108ed1c6c9b9b7849538c2e63d052ec1c01_a03465591aab9088b0f79bf5ef3a6e88, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,010 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ed1c6c9b9b7849538c2e63d052ec1c01_a03465591aab9088b0f79bf5ef3a6e88 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742212_1388 (size=12595) 2024-11-08T02:24:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742213_1389 (size=4469) 2024-11-08T02:24:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T02:24:19,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:19,044 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T02:24:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d9335d6bd5534610b30fc4f2470394cc_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032657701/Put/seqid=0 2024-11-08T02:24:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742214_1390 (size=9814) 2024-11-08T02:24:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,117 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d9335d6bd5534610b30fc4f2470394cc_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d9335d6bd5534610b30fc4f2470394cc_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/054f6a6992204943822cc8793aa52d03, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/054f6a6992204943822cc8793aa52d03 is 175, key is test_row_0/A:col10/1731032657701/Put/seqid=0 2024-11-08T02:24:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742215_1391 (size=22461) 2024-11-08T02:24:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T02:24:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,418 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/15ad8817d38f4c01867f4fef078a74b0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/15ad8817d38f4c01867f4fef078a74b0 2024-11-08T02:24:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,423 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/B of a03465591aab9088b0f79bf5ef3a6e88 into 15ad8817d38f4c01867f4fef078a74b0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:19,423 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,423 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/B, priority=13, startTime=1731032658979; duration=0sec 2024-11-08T02:24:19,423 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:19,423 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:B 2024-11-08T02:24:19,423 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,424 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,424 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/665208f498e14360841df71115744599 is 175, key is test_row_0/A:col10/1731032657692/Put/seqid=0 2024-11-08T02:24:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,426 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:19,426 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/C is initiating minor compaction (all files) 2024-11-08T02:24:19,426 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/C in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:19,426 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/c4f1a5fa410f401e919215179ac26dd0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/5e81adad23004a848f93a486df98cf95, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/d12172bfaa354d1683baca6a5940b9d0] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=35.9 K 2024-11-08T02:24:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,427 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4f1a5fa410f401e919215179ac26dd0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731032653133 2024-11-08T02:24:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,427 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e81adad23004a848f93a486df98cf95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731032654297 2024-11-08T02:24:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,427 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting d12172bfaa354d1683baca6a5940b9d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731032656527 2024-11-08T02:24:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,446 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,447 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/0af0a13d8d9843a2a7c96fab4ebb66a2 is 50, key is test_row_0/C:col10/1731032657692/Put/seqid=0 2024-11-08T02:24:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742216_1392 (size=31549) 2024-11-08T02:24:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742217_1393 (size=12595) 2024-11-08T02:24:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,486 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/0af0a13d8d9843a2a7c96fab4ebb66a2 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/0af0a13d8d9843a2a7c96fab4ebb66a2 2024-11-08T02:24:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,491 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/C of a03465591aab9088b0f79bf5ef3a6e88 into 0af0a13d8d9843a2a7c96fab4ebb66a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:19,491 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:19,491 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/C, priority=13, startTime=1731032658979; duration=0sec 2024-11-08T02:24:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,491 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:19,491 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:C 2024-11-08T02:24:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T02:24:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,525 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/054f6a6992204943822cc8793aa52d03 2024-11-08T02:24:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/8ec6afc29857464baf214ea936a482e2 is 50, key is test_row_0/B:col10/1731032657701/Put/seqid=0 2024-11-08T02:24:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742218_1394 (size=9757) 2024-11-08T02:24:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,541 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/8ec6afc29857464baf214ea936a482e2 2024-11-08T02:24:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/e2e6d5fb4ec44bf49c5cc899e7b0683a is 50, key is test_row_0/C:col10/1731032657701/Put/seqid=0 2024-11-08T02:24:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742219_1395 (size=9757) 2024-11-08T02:24:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,882 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/665208f498e14360841df71115744599 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/665208f498e14360841df71115744599 2024-11-08T02:24:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,887 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/A of a03465591aab9088b0f79bf5ef3a6e88 into 665208f498e14360841df71115744599(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:19,887 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:19,887 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/A, priority=13, startTime=1731032658979; duration=0sec 2024-11-08T02:24:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,887 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:19,887 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:A 2024-11-08T02:24:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:19,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,957 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/e2e6d5fb4ec44bf49c5cc899e7b0683a 2024-11-08T02:24:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/054f6a6992204943822cc8793aa52d03 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/054f6a6992204943822cc8793aa52d03 2024-11-08T02:24:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,967 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/054f6a6992204943822cc8793aa52d03, entries=100, sequenceid=205, filesize=21.9 K 2024-11-08T02:24:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/8ec6afc29857464baf214ea936a482e2 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/8ec6afc29857464baf214ea936a482e2 2024-11-08T02:24:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,972 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/8ec6afc29857464baf214ea936a482e2, entries=100, sequenceid=205, filesize=9.5 K 2024-11-08T02:24:19,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/e2e6d5fb4ec44bf49c5cc899e7b0683a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/e2e6d5fb4ec44bf49c5cc899e7b0683a 2024-11-08T02:24:19,978 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/e2e6d5fb4ec44bf49c5cc899e7b0683a, entries=100, sequenceid=205, filesize=9.5 K 2024-11-08T02:24:19,979 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=114.05 KB/116790 for a03465591aab9088b0f79bf5ef3a6e88 in 935ms, sequenceid=205, compaction requested=false 2024-11-08T02:24:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:19,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:19,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-08T02:24:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-08T02:24:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:19,981 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:19,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T02:24:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-08T02:24:19,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0910 sec 2024-11-08T02:24:19,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.0990 sec 2024-11-08T02:24:19,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082847bafa6f1f4799bb88a3cb8ff1f3d0_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032659958/Put/seqid=0 2024-11-08T02:24:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T02:24:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,029 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:20,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-08T02:24:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,034 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T02:24:20,039 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:20,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:20,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032720028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032720036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032720041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032720041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742221_1397 (size=49662) 2024-11-08T02:24:20,061 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:20,065 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082847bafa6f1f4799bb88a3cb8ff1f3d0_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082847bafa6f1f4799bb88a3cb8ff1f3d0_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:20,066 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5ad41fd9fe624c04b6571486ff8e9d41, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:20,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5ad41fd9fe624c04b6571486ff8e9d41 is 175, key is test_row_0/A:col10/1731032659958/Put/seqid=0 2024-11-08T02:24:20,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742220_1396 (size=160893) 2024-11-08T02:24:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T02:24:20,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032720148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032720148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032720158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032720158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:20,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:20,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:20,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:20,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T02:24:20,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032720358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032720358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032720368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032720368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,480 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5ad41fd9fe624c04b6571486ff8e9d41 2024-11-08T02:24:20,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/07a8a472987845a09fde2782261be599 is 50, key is test_row_0/B:col10/1731032659958/Put/seqid=0 2024-11-08T02:24:20,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742222_1398 (size=12151) 2024-11-08T02:24:20,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:20,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:20,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T02:24:20,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032720668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032720678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032720678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032720688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:20,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:20,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:20,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:20,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/07a8a472987845a09fde2782261be599 2024-11-08T02:24:20,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/6c5bdc5e7a23436d870090a59ba79515 is 50, key is test_row_0/C:col10/1731032659958/Put/seqid=0 2024-11-08T02:24:20,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742223_1399 (size=12151) 2024-11-08T02:24:20,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:20,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:20,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:20,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:21,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:21,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T02:24:21,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032721188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:21,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032721198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:21,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032721198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032721200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:21,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:21,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:21,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/6c5bdc5e7a23436d870090a59ba79515 2024-11-08T02:24:21,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5ad41fd9fe624c04b6571486ff8e9d41 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5ad41fd9fe624c04b6571486ff8e9d41 2024-11-08T02:24:21,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5ad41fd9fe624c04b6571486ff8e9d41, entries=900, sequenceid=229, filesize=157.1 K 2024-11-08T02:24:21,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/07a8a472987845a09fde2782261be599 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/07a8a472987845a09fde2782261be599 2024-11-08T02:24:21,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/07a8a472987845a09fde2782261be599, entries=150, sequenceid=229, filesize=11.9 K 2024-11-08T02:24:21,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/6c5bdc5e7a23436d870090a59ba79515 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6c5bdc5e7a23436d870090a59ba79515 2024-11-08T02:24:21,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6c5bdc5e7a23436d870090a59ba79515, entries=150, sequenceid=229, filesize=11.9 K 2024-11-08T02:24:21,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a03465591aab9088b0f79bf5ef3a6e88 in 1345ms, sequenceid=229, compaction requested=true 2024-11-08T02:24:21,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:21,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:21,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:21,327 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:21,327 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:21,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:21,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:21,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:21,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:21,328 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 214903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:21,328 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:21,328 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/A is initiating minor compaction (all files) 2024-11-08T02:24:21,328 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/B is initiating minor compaction (all files) 2024-11-08T02:24:21,328 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/A in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,328 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/B in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,328 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/665208f498e14360841df71115744599, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/054f6a6992204943822cc8793aa52d03, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5ad41fd9fe624c04b6571486ff8e9d41] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=209.9 K 2024-11-08T02:24:21,328 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/15ad8817d38f4c01867f4fef078a74b0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/8ec6afc29857464baf214ea936a482e2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/07a8a472987845a09fde2782261be599] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=33.7 K 2024-11-08T02:24:21,328 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,328 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/665208f498e14360841df71115744599, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/054f6a6992204943822cc8793aa52d03, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5ad41fd9fe624c04b6571486ff8e9d41] 2024-11-08T02:24:21,328 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 15ad8817d38f4c01867f4fef078a74b0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731032656527 2024-11-08T02:24:21,328 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 665208f498e14360841df71115744599, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731032656527 2024-11-08T02:24:21,328 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ec6afc29857464baf214ea936a482e2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731032657701 2024-11-08T02:24:21,329 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 054f6a6992204943822cc8793aa52d03, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731032657701 2024-11-08T02:24:21,329 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 07a8a472987845a09fde2782261be599, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731032659943 2024-11-08T02:24:21,329 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ad41fd9fe624c04b6571486ff8e9d41, keycount=900, bloomtype=ROW, size=157.1 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731032659891 2024-11-08T02:24:21,333 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:21,334 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:21,335 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/06d3082b187e49c892668f2133ab9106 is 50, key is test_row_0/B:col10/1731032659958/Put/seqid=0 2024-11-08T02:24:21,340 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108b7740cbca749427cabe409e276a79b28_a03465591aab9088b0f79bf5ef3a6e88 store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:21,344 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108b7740cbca749427cabe409e276a79b28_a03465591aab9088b0f79bf5ef3a6e88, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:21,344 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b7740cbca749427cabe409e276a79b28_a03465591aab9088b0f79bf5ef3a6e88 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:21,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742224_1400 (size=12697) 2024-11-08T02:24:21,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742225_1401 (size=4469) 2024-11-08T02:24:21,357 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#A#compaction#340 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:21,358 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/6c0f1cc373564d7abc74823546a7f63e is 175, key is test_row_0/A:col10/1731032659958/Put/seqid=0 2024-11-08T02:24:21,361 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/06d3082b187e49c892668f2133ab9106 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/06d3082b187e49c892668f2133ab9106 2024-11-08T02:24:21,368 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/B of a03465591aab9088b0f79bf5ef3a6e88 into 06d3082b187e49c892668f2133ab9106(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:21,368 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:21,368 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/B, priority=13, startTime=1731032661327; duration=0sec 2024-11-08T02:24:21,368 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:21,368 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:B 2024-11-08T02:24:21,368 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:21,369 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:21,369 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/C is initiating minor compaction (all files) 2024-11-08T02:24:21,369 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/C in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,369 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/0af0a13d8d9843a2a7c96fab4ebb66a2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/e2e6d5fb4ec44bf49c5cc899e7b0683a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6c5bdc5e7a23436d870090a59ba79515] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=33.7 K 2024-11-08T02:24:21,370 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 0af0a13d8d9843a2a7c96fab4ebb66a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731032656527 2024-11-08T02:24:21,370 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting e2e6d5fb4ec44bf49c5cc899e7b0683a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731032657701 2024-11-08T02:24:21,370 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c5bdc5e7a23436d870090a59ba79515, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731032659943 2024-11-08T02:24:21,378 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#C#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:21,379 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/a2c454e0e4de4436a4503dc4261da877 is 50, key is test_row_0/C:col10/1731032659958/Put/seqid=0 2024-11-08T02:24:21,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742226_1402 (size=31651) 2024-11-08T02:24:21,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742227_1403 (size=12697) 2024-11-08T02:24:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:21,403 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:21,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T02:24:21,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108db31851ffa70423482ff58ed347bd3b2_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032660036/Put/seqid=0 2024-11-08T02:24:21,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:21,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:21,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742228_1404 (size=14794) 2024-11-08T02:24:21,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:21,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:21,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032721580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:21,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032721698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:21,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:21,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:21,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,790 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/6c0f1cc373564d7abc74823546a7f63e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/6c0f1cc373564d7abc74823546a7f63e 2024-11-08T02:24:21,796 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/A of a03465591aab9088b0f79bf5ef3a6e88 into 6c0f1cc373564d7abc74823546a7f63e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:21,796 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:21,796 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/A, priority=13, startTime=1731032661327; duration=0sec 2024-11-08T02:24:21,797 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:21,797 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:A 2024-11-08T02:24:21,804 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/a2c454e0e4de4436a4503dc4261da877 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/a2c454e0e4de4436a4503dc4261da877 2024-11-08T02:24:21,811 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/C of a03465591aab9088b0f79bf5ef3a6e88 into a2c454e0e4de4436a4503dc4261da877(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:21,811 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:21,811 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/C, priority=13, startTime=1731032661327; duration=0sec 2024-11-08T02:24:21,811 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:21,811 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:C 2024-11-08T02:24:21,834 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:21,837 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108db31851ffa70423482ff58ed347bd3b2_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108db31851ffa70423482ff58ed347bd3b2_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:21,839 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5da2ffa88738456889035c4aa73e324d, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:21,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5da2ffa88738456889035c4aa73e324d is 175, key is test_row_0/A:col10/1731032660036/Put/seqid=0 2024-11-08T02:24:21,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742229_1405 (size=39749) 2024-11-08T02:24:21,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:21,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:21,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:21,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:21,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032721918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:22,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:22,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:22,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:22,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:22,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T02:24:22,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032722208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:22,213 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in a03465591aab9088b0f79bf5ef3a6e88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:22,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032722217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:22,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032722217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:22,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032722218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:22,223 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in a03465591aab9088b0f79bf5ef3a6e88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,223 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in a03465591aab9088b0f79bf5ef3a6e88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:22,223 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in a03465591aab9088b0f79bf5ef3a6e88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:22,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032722238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:22,244 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5da2ffa88738456889035c4aa73e324d 2024-11-08T02:24:22,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/1b55ac6fb2bc4c4fa54708c5e5420c9a is 50, key is test_row_0/B:col10/1731032660036/Put/seqid=0 2024-11-08T02:24:22,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742230_1406 (size=12151) 2024-11-08T02:24:22,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:22,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:22,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:22,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:22,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/1b55ac6fb2bc4c4fa54708c5e5420c9a 2024-11-08T02:24:22,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f3e94e1b956d49bf8b7a857d3fd6fce0 is 50, key is test_row_0/C:col10/1731032660036/Put/seqid=0 2024-11-08T02:24:22,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742231_1407 (size=12151) 2024-11-08T02:24:22,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f3e94e1b956d49bf8b7a857d3fd6fce0 2024-11-08T02:24:22,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/5da2ffa88738456889035c4aa73e324d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5da2ffa88738456889035c4aa73e324d 2024-11-08T02:24:22,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5da2ffa88738456889035c4aa73e324d, entries=200, sequenceid=246, filesize=38.8 K 2024-11-08T02:24:22,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/1b55ac6fb2bc4c4fa54708c5e5420c9a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/1b55ac6fb2bc4c4fa54708c5e5420c9a 2024-11-08T02:24:22,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/1b55ac6fb2bc4c4fa54708c5e5420c9a, entries=150, sequenceid=246, filesize=11.9 K 2024-11-08T02:24:22,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f3e94e1b956d49bf8b7a857d3fd6fce0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f3e94e1b956d49bf8b7a857d3fd6fce0 2024-11-08T02:24:22,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f3e94e1b956d49bf8b7a857d3fd6fce0, entries=150, sequenceid=246, filesize=11.9 K 2024-11-08T02:24:22,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a03465591aab9088b0f79bf5ef3a6e88 in 1276ms, sequenceid=246, compaction requested=false 2024-11-08T02:24:22,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:22,752 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:22,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T02:24:22,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e677de14aeba4b42b2a4945be5fe9536_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032661561/Put/seqid=0 2024-11-08T02:24:22,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742232_1408 (size=14994) 2024-11-08T02:24:22,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:22,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:22,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032722814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:22,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032722928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:22,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:22,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:22,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:22,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:23,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:23,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:23,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:23,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032723138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:23,162 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e677de14aeba4b42b2a4945be5fe9536_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e677de14aeba4b42b2a4945be5fe9536_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:23,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/84098ec435ee49adb242d010c5f8297f, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:23,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/84098ec435ee49adb242d010c5f8297f is 175, key is test_row_0/A:col10/1731032661561/Put/seqid=0 2024-11-08T02:24:23,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742233_1409 (size=39949) 2024-11-08T02:24:23,171 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/84098ec435ee49adb242d010c5f8297f 2024-11-08T02:24:23,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/6435d3a4959740b491b3e953a28703d2 is 50, key is test_row_0/B:col10/1731032661561/Put/seqid=0 2024-11-08T02:24:23,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742234_1410 (size=12301) 2024-11-08T02:24:23,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/6435d3a4959740b491b3e953a28703d2 2024-11-08T02:24:23,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f6a9cde4a9a145149d833a4e3eea3729 is 50, key is test_row_0/C:col10/1731032661561/Put/seqid=0 2024-11-08T02:24:23,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742235_1411 (size=12301) 2024-11-08T02:24:23,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f6a9cde4a9a145149d833a4e3eea3729 2024-11-08T02:24:23,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/84098ec435ee49adb242d010c5f8297f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/84098ec435ee49adb242d010c5f8297f 2024-11-08T02:24:23,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/84098ec435ee49adb242d010c5f8297f, entries=200, sequenceid=269, filesize=39.0 K 2024-11-08T02:24:23,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/6435d3a4959740b491b3e953a28703d2 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6435d3a4959740b491b3e953a28703d2 2024-11-08T02:24:23,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6435d3a4959740b491b3e953a28703d2, entries=150, sequenceid=269, filesize=12.0 K 2024-11-08T02:24:23,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f6a9cde4a9a145149d833a4e3eea3729 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f6a9cde4a9a145149d833a4e3eea3729 2024-11-08T02:24:23,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f6a9cde4a9a145149d833a4e3eea3729, entries=150, sequenceid=269, filesize=12.0 K 2024-11-08T02:24:23,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a03465591aab9088b0f79bf5ef3a6e88 in 454ms, sequenceid=269, compaction requested=true 2024-11-08T02:24:23,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:23,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:23,206 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:23,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:23,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:23,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:23,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:23,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T02:24:23,206 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:23,209 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:23,209 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:23,209 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/B is initiating minor compaction (all files) 2024-11-08T02:24:23,209 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/A is initiating minor compaction (all files) 2024-11-08T02:24:23,209 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/A in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:23,209 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/B in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:23,209 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/6c0f1cc373564d7abc74823546a7f63e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5da2ffa88738456889035c4aa73e324d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/84098ec435ee49adb242d010c5f8297f] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=108.7 K 2024-11-08T02:24:23,209 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/06d3082b187e49c892668f2133ab9106, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/1b55ac6fb2bc4c4fa54708c5e5420c9a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6435d3a4959740b491b3e953a28703d2] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=36.3 K 2024-11-08T02:24:23,209 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:23,209 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/6c0f1cc373564d7abc74823546a7f63e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5da2ffa88738456889035c4aa73e324d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/84098ec435ee49adb242d010c5f8297f] 2024-11-08T02:24:23,209 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06d3082b187e49c892668f2133ab9106, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731032659943 2024-11-08T02:24:23,210 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c0f1cc373564d7abc74823546a7f63e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731032659943 2024-11-08T02:24:23,210 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b55ac6fb2bc4c4fa54708c5e5420c9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731032660026 2024-11-08T02:24:23,210 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 5da2ffa88738456889035c4aa73e324d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731032660026 2024-11-08T02:24:23,210 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6435d3a4959740b491b3e953a28703d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731032661556 2024-11-08T02:24:23,210 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 84098ec435ee49adb242d010c5f8297f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731032661556 2024-11-08T02:24:23,215 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:23,216 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:23,216 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/9dd8da676fed44889f6280ee9f52645c is 50, key is test_row_0/B:col10/1731032661561/Put/seqid=0 2024-11-08T02:24:23,221 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110809c7d614439c4f25abcf477247d20069_a03465591aab9088b0f79bf5ef3a6e88 store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:23,223 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110809c7d614439c4f25abcf477247d20069_a03465591aab9088b0f79bf5ef3a6e88, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:23,223 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110809c7d614439c4f25abcf477247d20069_a03465591aab9088b0f79bf5ef3a6e88 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:23,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742236_1412 (size=12949) 2024-11-08T02:24:23,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742237_1413 (size=4469) 2024-11-08T02:24:23,235 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#A#compaction#349 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:23,236 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/b3f70c0b0d5f4464b789f567aa982021 is 175, key is test_row_0/A:col10/1731032661561/Put/seqid=0 2024-11-08T02:24:23,239 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/9dd8da676fed44889f6280ee9f52645c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/9dd8da676fed44889f6280ee9f52645c 2024-11-08T02:24:23,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742238_1414 (size=31903) 2024-11-08T02:24:23,253 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/B of a03465591aab9088b0f79bf5ef3a6e88 into 9dd8da676fed44889f6280ee9f52645c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:23,253 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:23,253 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/B, priority=13, startTime=1731032663206; duration=0sec 2024-11-08T02:24:23,254 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:23,254 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:B 2024-11-08T02:24:23,254 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:23,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T02:24:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:23,255 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T02:24:23,255 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:23,255 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/C is initiating minor compaction (all files) 2024-11-08T02:24:23,255 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/C in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:23,255 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/a2c454e0e4de4436a4503dc4261da877, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f3e94e1b956d49bf8b7a857d3fd6fce0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f6a9cde4a9a145149d833a4e3eea3729] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=36.3 K 2024-11-08T02:24:23,256 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c454e0e4de4436a4503dc4261da877, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731032659943 2024-11-08T02:24:23,257 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3e94e1b956d49bf8b7a857d3fd6fce0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731032660026 2024-11-08T02:24:23,259 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6a9cde4a9a145149d833a4e3eea3729, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731032661556 2024-11-08T02:24:23,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082f57579d6b9f44689503689afd1ce4b9_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032662802/Put/seqid=0 2024-11-08T02:24:23,270 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#C#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:23,270 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/6ec70a3b50264cf7bc752be3cd456a14 is 50, key is test_row_0/C:col10/1731032661561/Put/seqid=0 2024-11-08T02:24:23,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742239_1415 (size=12454) 2024-11-08T02:24:23,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,288 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082f57579d6b9f44689503689afd1ce4b9_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082f57579d6b9f44689503689afd1ce4b9_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:23,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/da212e0278904827ac30a0130b21d200, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:23,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/da212e0278904827ac30a0130b21d200 is 175, key is test_row_0/A:col10/1731032662802/Put/seqid=0 2024-11-08T02:24:23,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742240_1416 (size=12949) 2024-11-08T02:24:23,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742241_1417 (size=31255) 2024-11-08T02:24:23,310 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/da212e0278904827ac30a0130b21d200 2024-11-08T02:24:23,316 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/6ec70a3b50264cf7bc752be3cd456a14 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6ec70a3b50264cf7bc752be3cd456a14 2024-11-08T02:24:23,324 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/C of a03465591aab9088b0f79bf5ef3a6e88 into 6ec70a3b50264cf7bc752be3cd456a14(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:23,324 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:23,324 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/C, priority=13, startTime=1731032663206; duration=0sec 2024-11-08T02:24:23,324 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:23,324 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:C 2024-11-08T02:24:23,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/56d461a272494d60828b1849477dab9d is 50, key is test_row_0/B:col10/1731032662802/Put/seqid=0 2024-11-08T02:24:23,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742242_1418 (size=12301) 2024-11-08T02:24:23,341 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/56d461a272494d60828b1849477dab9d 2024-11-08T02:24:23,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f80a94640a774bc0bd12364256fb1330 is 50, key is test_row_0/C:col10/1731032662802/Put/seqid=0 2024-11-08T02:24:23,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742243_1419 (size=12301) 2024-11-08T02:24:23,356 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f80a94640a774bc0bd12364256fb1330 2024-11-08T02:24:23,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/da212e0278904827ac30a0130b21d200 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/da212e0278904827ac30a0130b21d200 2024-11-08T02:24:23,368 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/da212e0278904827ac30a0130b21d200, entries=150, sequenceid=285, filesize=30.5 K 2024-11-08T02:24:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/56d461a272494d60828b1849477dab9d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/56d461a272494d60828b1849477dab9d 2024-11-08T02:24:23,373 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/56d461a272494d60828b1849477dab9d, entries=150, sequenceid=285, filesize=12.0 K 2024-11-08T02:24:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/f80a94640a774bc0bd12364256fb1330 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f80a94640a774bc0bd12364256fb1330 2024-11-08T02:24:23,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,380 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f80a94640a774bc0bd12364256fb1330, entries=150, sequenceid=285, filesize=12.0 K 2024-11-08T02:24:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,381 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for a03465591aab9088b0f79bf5ef3a6e88 in 127ms, sequenceid=285, compaction requested=false 2024-11-08T02:24:23,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:23,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:23,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-08T02:24:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-08T02:24:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-08T02:24:23,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3440 sec 2024-11-08T02:24:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 3.3570 sec 2024-11-08T02:24:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,647 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/b3f70c0b0d5f4464b789f567aa982021 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b3f70c0b0d5f4464b789f567aa982021 2024-11-08T02:24:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,653 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/A of a03465591aab9088b0f79bf5ef3a6e88 into b3f70c0b0d5f4464b789f567aa982021(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:23,653 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:23,653 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/A, priority=13, startTime=1731032663206; duration=0sec 2024-11-08T02:24:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,653 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:23,653 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:A 2024-11-08T02:24:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:23,726 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:23,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:24:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108feef677c56524f17aa2c57acaa99b58e_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032663724/Put/seqid=0 2024-11-08T02:24:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742244_1420 (size=12454) 2024-11-08T02:24:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,742 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,746 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108feef677c56524f17aa2c57acaa99b58e_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108feef677c56524f17aa2c57acaa99b58e_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,747 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/74dc4593b87c4bb7b370516b5c7e5c34, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/74dc4593b87c4bb7b370516b5c7e5c34 is 175, key is test_row_0/A:col10/1731032663724/Put/seqid=0 2024-11-08T02:24:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742245_1421 (size=31251) 2024-11-08T02:24:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,764 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/74dc4593b87c4bb7b370516b5c7e5c34 2024-11-08T02:24:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/9fc3c346e325499a88b63b39ba23d48b is 50, key is test_row_0/B:col10/1731032663724/Put/seqid=0 2024-11-08T02:24:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742246_1422 (size=9857) 2024-11-08T02:24:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:24,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032724046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:24,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032724173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T02:24:24,198 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:24,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-08T02:24:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T02:24:24,200 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:24,201 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:24,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:24,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/9fc3c346e325499a88b63b39ba23d48b 2024-11-08T02:24:24,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40502 deadline: 1731032724228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:24,231 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in a03465591aab9088b0f79bf5ef3a6e88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:24,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/a6992e2dab054c62b3f91f43c8365ff6 is 50, key is test_row_0/C:col10/1731032663724/Put/seqid=0 2024-11-08T02:24:24,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742247_1423 (size=9857) 2024-11-08T02:24:24,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:24,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40430 deadline: 1731032724238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:24,244 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in a03465591aab9088b0f79bf5ef3a6e88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:24,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:24,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40470 deadline: 1731032724247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:24,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731032724247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:24,252 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in a03465591aab9088b0f79bf5ef3a6e88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:24,252 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in a03465591aab9088b0f79bf5ef3a6e88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T02:24:24,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-08T02:24:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:24,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:24,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032724388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:24,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-08T02:24:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:24,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T02:24:24,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/a6992e2dab054c62b3f91f43c8365ff6 2024-11-08T02:24:24,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/74dc4593b87c4bb7b370516b5c7e5c34 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/74dc4593b87c4bb7b370516b5c7e5c34 2024-11-08T02:24:24,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/74dc4593b87c4bb7b370516b5c7e5c34, entries=150, sequenceid=298, filesize=30.5 K 2024-11-08T02:24:24,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/9fc3c346e325499a88b63b39ba23d48b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/9fc3c346e325499a88b63b39ba23d48b 2024-11-08T02:24:24,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/9fc3c346e325499a88b63b39ba23d48b, entries=100, sequenceid=298, filesize=9.6 K 2024-11-08T02:24:24,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/a6992e2dab054c62b3f91f43c8365ff6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/a6992e2dab054c62b3f91f43c8365ff6 2024-11-08T02:24:24,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/a6992e2dab054c62b3f91f43c8365ff6, entries=100, sequenceid=298, filesize=9.6 K 2024-11-08T02:24:24,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a03465591aab9088b0f79bf5ef3a6e88 in 928ms, sequenceid=298, compaction requested=true 2024-11-08T02:24:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:24,655 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:24,655 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:24,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:24,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:24,656 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:24,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:24,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:24,656 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/A is initiating minor compaction (all files) 2024-11-08T02:24:24,656 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/A in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:24,656 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b3f70c0b0d5f4464b789f567aa982021, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/da212e0278904827ac30a0130b21d200, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/74dc4593b87c4bb7b370516b5c7e5c34] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=92.2 K 2024-11-08T02:24:24,656 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:24,656 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b3f70c0b0d5f4464b789f567aa982021, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/da212e0278904827ac30a0130b21d200, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/74dc4593b87c4bb7b370516b5c7e5c34] 2024-11-08T02:24:24,657 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3f70c0b0d5f4464b789f567aa982021, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731032661556 2024-11-08T02:24:24,657 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:24,657 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/B is initiating minor compaction (all files) 2024-11-08T02:24:24,657 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/B in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:24,657 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/9dd8da676fed44889f6280ee9f52645c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/56d461a272494d60828b1849477dab9d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/9fc3c346e325499a88b63b39ba23d48b] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=34.3 K 2024-11-08T02:24:24,657 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting da212e0278904827ac30a0130b21d200, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731032662791 2024-11-08T02:24:24,658 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dd8da676fed44889f6280ee9f52645c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731032661556 2024-11-08T02:24:24,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-08T02:24:24,658 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74dc4593b87c4bb7b370516b5c7e5c34, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731032663676 2024-11-08T02:24:24,658 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 56d461a272494d60828b1849477dab9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731032662791 2024-11-08T02:24:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:24,658 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:24:24,658 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fc3c346e325499a88b63b39ba23d48b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731032663695 2024-11-08T02:24:24,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110895332f38a63047fa800b8e51f23dfc7e_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032664040/Put/seqid=0 2024-11-08T02:24:24,676 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:24,677 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#B#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:24,677 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/f9e8d8aa4a6f4f08951ecf0ce5566036 is 50, key is test_row_0/B:col10/1731032663724/Put/seqid=0 2024-11-08T02:24:24,678 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108f92cc98d2f36412b8ddcbf235477e864_a03465591aab9088b0f79bf5ef3a6e88 store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:24,680 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108f92cc98d2f36412b8ddcbf235477e864_a03465591aab9088b0f79bf5ef3a6e88, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:24,680 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f92cc98d2f36412b8ddcbf235477e864_a03465591aab9088b0f79bf5ef3a6e88 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:24,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742248_1424 (size=12454) 2024-11-08T02:24:24,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742249_1425 (size=13051) 2024-11-08T02:24:24,706 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/f9e8d8aa4a6f4f08951ecf0ce5566036 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/f9e8d8aa4a6f4f08951ecf0ce5566036 2024-11-08T02:24:24,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. as already flushing 2024-11-08T02:24:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:24,712 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/B of a03465591aab9088b0f79bf5ef3a6e88 into f9e8d8aa4a6f4f08951ecf0ce5566036(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:24,712 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:24,712 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/B, priority=13, startTime=1731032664655; duration=0sec 2024-11-08T02:24:24,713 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:24,713 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:B 2024-11-08T02:24:24,713 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:24,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742250_1426 (size=4469) 2024-11-08T02:24:24,714 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:24,714 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): a03465591aab9088b0f79bf5ef3a6e88/C is initiating minor compaction (all files) 2024-11-08T02:24:24,714 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03465591aab9088b0f79bf5ef3a6e88/C in TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:24,714 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6ec70a3b50264cf7bc752be3cd456a14, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f80a94640a774bc0bd12364256fb1330, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/a6992e2dab054c62b3f91f43c8365ff6] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp, totalSize=34.3 K 2024-11-08T02:24:24,715 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ec70a3b50264cf7bc752be3cd456a14, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731032661556 2024-11-08T02:24:24,715 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting f80a94640a774bc0bd12364256fb1330, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731032662791 2024-11-08T02:24:24,715 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a6992e2dab054c62b3f91f43c8365ff6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731032663695 2024-11-08T02:24:24,730 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#C#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:24,730 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/6390a5bc134f41a39c2ab304c80bbda3 is 50, key is test_row_0/C:col10/1731032663724/Put/seqid=0 2024-11-08T02:24:24,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742251_1427 (size=13051) 2024-11-08T02:24:24,745 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/6390a5bc134f41a39c2ab304c80bbda3 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6390a5bc134f41a39c2ab304c80bbda3 2024-11-08T02:24:24,751 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/C of a03465591aab9088b0f79bf5ef3a6e88 into 6390a5bc134f41a39c2ab304c80bbda3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:24,751 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:24,752 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/C, priority=13, startTime=1731032664656; duration=0sec 2024-11-08T02:24:24,752 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:24,752 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:C 2024-11-08T02:24:24,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032724773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T02:24:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032724888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:25,093 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110895332f38a63047fa800b8e51f23dfc7e_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110895332f38a63047fa800b8e51f23dfc7e_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:25,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/20e3932cc7cb4374925714206f85e770, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:25,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/20e3932cc7cb4374925714206f85e770 is 175, key is test_row_0/A:col10/1731032664040/Put/seqid=0 2024-11-08T02:24:25,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742252_1428 (size=31255) 2024-11-08T02:24:25,100 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/20e3932cc7cb4374925714206f85e770 2024-11-08T02:24:25,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/429d601b983b453fa7030c7803424b72 is 50, key is test_row_0/B:col10/1731032664040/Put/seqid=0 2024-11-08T02:24:25,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:25,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032725098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:25,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:25,114 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03465591aab9088b0f79bf5ef3a6e88#A#compaction#359 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:25,115 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/f916a7f3daec4817a7aaf5fb1ded1d9d is 175, key is test_row_0/A:col10/1731032663724/Put/seqid=0 2024-11-08T02:24:25,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742253_1429 (size=12301) 2024-11-08T02:24:25,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742254_1430 (size=32112) 2024-11-08T02:24:25,137 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/f916a7f3daec4817a7aaf5fb1ded1d9d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/f916a7f3daec4817a7aaf5fb1ded1d9d 2024-11-08T02:24:25,141 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03465591aab9088b0f79bf5ef3a6e88/A of a03465591aab9088b0f79bf5ef3a6e88 into f916a7f3daec4817a7aaf5fb1ded1d9d(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:25,141 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:25,141 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., storeName=a03465591aab9088b0f79bf5ef3a6e88/A, priority=13, startTime=1731032664655; duration=0sec 2024-11-08T02:24:25,141 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:25,141 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:A 2024-11-08T02:24:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T02:24:25,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032725418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:25,518 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/429d601b983b453fa7030c7803424b72 2024-11-08T02:24:25,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/1a4b1b8c35594ab6a382b886529d5bba is 50, key is test_row_0/C:col10/1731032664040/Put/seqid=0 2024-11-08T02:24:25,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742255_1431 (size=12301) 2024-11-08T02:24:25,927 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/1a4b1b8c35594ab6a382b886529d5bba 2024-11-08T02:24:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/20e3932cc7cb4374925714206f85e770 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/20e3932cc7cb4374925714206f85e770 2024-11-08T02:24:25,934 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/20e3932cc7cb4374925714206f85e770, entries=150, sequenceid=323, filesize=30.5 K 2024-11-08T02:24:25,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40486 deadline: 1731032725928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:25,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/429d601b983b453fa7030c7803424b72 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/429d601b983b453fa7030c7803424b72 2024-11-08T02:24:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:24:25,940 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/429d601b983b453fa7030c7803424b72, entries=150, sequenceid=323, filesize=12.0 K 2024-11-08T02:24:25,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/1a4b1b8c35594ab6a382b886529d5bba as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/1a4b1b8c35594ab6a382b886529d5bba 2024-11-08T02:24:25,945 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/1a4b1b8c35594ab6a382b886529d5bba, entries=150, sequenceid=323, filesize=12.0 K 2024-11-08T02:24:25,946 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a03465591aab9088b0f79bf5ef3a6e88 in 1288ms, sequenceid=323, compaction requested=false 2024-11-08T02:24:25,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:25,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:25,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-08T02:24:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-08T02:24:25,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-08T02:24:25,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7460 sec 2024-11-08T02:24:25,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.7530 sec 2024-11-08T02:24:26,264 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-11-08T02:24:26,264 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:26,264 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:26,265 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,265 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-11-08T02:24:26,265 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:26,265 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,266 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:26,266 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-08T02:24:26,266 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:26,266 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,266 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:26,269 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-08T02:24:26,269 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:26,269 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,270 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:26,272 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-08T02:24:26,272 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:26,272 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,272 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T02:24:26,348 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:26,570 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:24:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:26,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:24:26,950 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-11-08T02:24:26,950 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:26,950 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,950 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:26,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:26,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108df7f2ee226a34d33a1c5b4d558d1eca8_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032664771/Put/seqid=0 2024-11-08T02:24:26,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742256_1432 (size=12454) 2024-11-08T02:24:27,359 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:27,362 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108df7f2ee226a34d33a1c5b4d558d1eca8_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108df7f2ee226a34d33a1c5b4d558d1eca8_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:27,363 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/71b665ccecf340abb658de9269cff5fe, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:27,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/71b665ccecf340abb658de9269cff5fe is 175, key is test_row_0/A:col10/1731032664771/Put/seqid=0 2024-11-08T02:24:27,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742257_1433 (size=31255) 2024-11-08T02:24:27,768 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/71b665ccecf340abb658de9269cff5fe 2024-11-08T02:24:27,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/ea43515a6b4246e583971b9b54939a45 is 50, key is test_row_0/B:col10/1731032664771/Put/seqid=0 2024-11-08T02:24:27,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742258_1434 (size=12301) 2024-11-08T02:24:28,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/ea43515a6b4246e583971b9b54939a45 2024-11-08T02:24:28,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/91f48a70a38745f68c6fe855d8882ddd is 50, key is test_row_0/C:col10/1731032664771/Put/seqid=0 2024-11-08T02:24:28,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742259_1435 (size=12301) 2024-11-08T02:24:28,251 INFO [Thread-1535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1535. 2024-11-08T02:24:28,251 DEBUG [Thread-1535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:28,251 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:28,251 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:28,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:28,260 INFO [Thread-1531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1531. 2024-11-08T02:24:28,260 DEBUG [Thread-1531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:28,260 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:28,260 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:28,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:28,271 INFO [Thread-1533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1533. 2024-11-08T02:24:28,271 DEBUG [Thread-1533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:28,271 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:28,271 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:28,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:28,291 INFO [Thread-1529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1529. 2024-11-08T02:24:28,292 DEBUG [Thread-1529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:28,292 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:28,292 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:28,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1392 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4151 rows 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1387 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4158 rows 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1385 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4151 rows 2024-11-08T02:24:28,293 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1390 2024-11-08T02:24:28,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4164 rows 2024-11-08T02:24:28,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1385 2024-11-08T02:24:28,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4154 rows 2024-11-08T02:24:28,294 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T02:24:28,294 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:24:28,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:28,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:28,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:28,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:24:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T02:24:28,300 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032668300"}]},"ts":"1731032668300"} 2024-11-08T02:24:28,301 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:24:28,301 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:24:28,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:24:28,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, UNASSIGN}] 2024-11-08T02:24:28,303 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, UNASSIGN 2024-11-08T02:24:28,304 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=a03465591aab9088b0f79bf5ef3a6e88, regionState=CLOSING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:24:28,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, UNASSIGN because future has completed 2024-11-08T02:24:28,305 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:24:28,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:24:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T02:24:28,457 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:28,458 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T02:24:28,458 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing a03465591aab9088b0f79bf5ef3a6e88, disabling compactions & flushes 2024-11-08T02:24:28,458 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:28,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/91f48a70a38745f68c6fe855d8882ddd 2024-11-08T02:24:28,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/71b665ccecf340abb658de9269cff5fe as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/71b665ccecf340abb658de9269cff5fe 2024-11-08T02:24:28,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/71b665ccecf340abb658de9269cff5fe, entries=150, sequenceid=338, filesize=30.5 K 2024-11-08T02:24:28,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/ea43515a6b4246e583971b9b54939a45 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/ea43515a6b4246e583971b9b54939a45 2024-11-08T02:24:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T02:24:28,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/ea43515a6b4246e583971b9b54939a45, entries=150, sequenceid=338, filesize=12.0 K 2024-11-08T02:24:28,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/91f48a70a38745f68c6fe855d8882ddd as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/91f48a70a38745f68c6fe855d8882ddd 2024-11-08T02:24:28,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/91f48a70a38745f68c6fe855d8882ddd, entries=150, sequenceid=338, filesize=12.0 K 2024-11-08T02:24:28,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=26.84 KB/27480 for a03465591aab9088b0f79bf5ef3a6e88 in 1678ms, sequenceid=338, compaction requested=true 2024-11-08T02:24:28,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03465591aab9088b0f79bf5ef3a6e88: 2024-11-08T02:24:28,627 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:28,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:28,627 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:28,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:28,627 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. after waiting 0 ms 2024-11-08T02:24:28,628 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:28,628 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing a03465591aab9088b0f79bf5ef3a6e88 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T02:24:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03465591aab9088b0f79bf5ef3a6e88:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:28,628 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. because compaction request was cancelled 2024-11-08T02:24:28,628 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. because compaction request was cancelled 2024-11-08T02:24:28,628 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:A 2024-11-08T02:24:28,628 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:B 2024-11-08T02:24:28,628 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. because compaction request was cancelled 2024-11-08T02:24:28,628 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03465591aab9088b0f79bf5ef3a6e88:C 2024-11-08T02:24:28,634 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086788c5487b114eb79607dc905480d46e_a03465591aab9088b0f79bf5ef3a6e88 is 50, key is test_row_0/A:col10/1731032668269/Put/seqid=0 2024-11-08T02:24:28,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742260_1436 (size=9914) 2024-11-08T02:24:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T02:24:29,040 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:29,049 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086788c5487b114eb79607dc905480d46e_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086788c5487b114eb79607dc905480d46e_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:29,050 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/a70daf3c2afb422b960420ee7b7490a0, store: [table=TestAcidGuarantees family=A region=a03465591aab9088b0f79bf5ef3a6e88] 2024-11-08T02:24:29,051 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/a70daf3c2afb422b960420ee7b7490a0 is 175, key is test_row_0/A:col10/1731032668269/Put/seqid=0 2024-11-08T02:24:29,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742261_1437 (size=22561) 2024-11-08T02:24:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T02:24:29,456 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/a70daf3c2afb422b960420ee7b7490a0 2024-11-08T02:24:29,469 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/1a4d842198f04449a67291b4f7423736 is 50, key is test_row_0/B:col10/1731032668269/Put/seqid=0 2024-11-08T02:24:29,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742262_1438 (size=9857) 2024-11-08T02:24:29,874 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/1a4d842198f04449a67291b4f7423736 2024-11-08T02:24:29,888 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/e02189abf73940be93733e4bc6f96eac is 50, key is test_row_0/C:col10/1731032668269/Put/seqid=0 2024-11-08T02:24:29,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742263_1439 (size=9857) 2024-11-08T02:24:30,296 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/e02189abf73940be93733e4bc6f96eac 2024-11-08T02:24:30,308 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/A/a70daf3c2afb422b960420ee7b7490a0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/a70daf3c2afb422b960420ee7b7490a0 2024-11-08T02:24:30,315 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/a70daf3c2afb422b960420ee7b7490a0, entries=100, sequenceid=345, filesize=22.0 K 2024-11-08T02:24:30,316 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/B/1a4d842198f04449a67291b4f7423736 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/1a4d842198f04449a67291b4f7423736 2024-11-08T02:24:30,321 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/1a4d842198f04449a67291b4f7423736, entries=100, sequenceid=345, filesize=9.6 K 2024-11-08T02:24:30,323 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/.tmp/C/e02189abf73940be93733e4bc6f96eac as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/e02189abf73940be93733e4bc6f96eac 2024-11-08T02:24:30,327 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/e02189abf73940be93733e4bc6f96eac, entries=100, sequenceid=345, filesize=9.6 K 2024-11-08T02:24:30,328 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a03465591aab9088b0f79bf5ef3a6e88 in 1700ms, sequenceid=345, compaction requested=true 2024-11-08T02:24:30,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/304946d655a345108723bf54ed202abf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/a826005de4ef4aeab8e32fa72e148140, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/e3fe1ad648c3443a8668226af7dd67d3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/ad66fda3508349f084863cdf86b77952, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/554333007a564f669ad42adc7b755acf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/c41a068134204d10a982b4237c372601, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5f1e7915f01f4b379eafda9c4f1ac8ca, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b642c3d133f64d9ca0bc66e0d33bb740, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/1a5a3a8317a245cbbb6fd727f115678e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/fc54fcf1e27244d3982fa90c64443ec8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/dda74695d723437fbb7c819c6f47f6e0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/3e5e226bca8e488da386874f36199ac0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/947c31d4da3144f783113d195e4ca5e8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/665208f498e14360841df71115744599, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/054f6a6992204943822cc8793aa52d03, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5ad41fd9fe624c04b6571486ff8e9d41, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/6c0f1cc373564d7abc74823546a7f63e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5da2ffa88738456889035c4aa73e324d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/84098ec435ee49adb242d010c5f8297f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b3f70c0b0d5f4464b789f567aa982021, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/da212e0278904827ac30a0130b21d200, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/74dc4593b87c4bb7b370516b5c7e5c34] to archive 2024-11-08T02:24:30,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:24:30,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/304946d655a345108723bf54ed202abf to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/304946d655a345108723bf54ed202abf 2024-11-08T02:24:30,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/a826005de4ef4aeab8e32fa72e148140 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/a826005de4ef4aeab8e32fa72e148140 2024-11-08T02:24:30,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/e3fe1ad648c3443a8668226af7dd67d3 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/e3fe1ad648c3443a8668226af7dd67d3 2024-11-08T02:24:30,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/ad66fda3508349f084863cdf86b77952 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/ad66fda3508349f084863cdf86b77952 2024-11-08T02:24:30,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/554333007a564f669ad42adc7b755acf to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/554333007a564f669ad42adc7b755acf 2024-11-08T02:24:30,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/c41a068134204d10a982b4237c372601 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/c41a068134204d10a982b4237c372601 2024-11-08T02:24:30,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5f1e7915f01f4b379eafda9c4f1ac8ca to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5f1e7915f01f4b379eafda9c4f1ac8ca 2024-11-08T02:24:30,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b642c3d133f64d9ca0bc66e0d33bb740 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b642c3d133f64d9ca0bc66e0d33bb740 2024-11-08T02:24:30,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/1a5a3a8317a245cbbb6fd727f115678e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/1a5a3a8317a245cbbb6fd727f115678e 2024-11-08T02:24:30,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/fc54fcf1e27244d3982fa90c64443ec8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/fc54fcf1e27244d3982fa90c64443ec8 2024-11-08T02:24:30,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/dda74695d723437fbb7c819c6f47f6e0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/dda74695d723437fbb7c819c6f47f6e0 2024-11-08T02:24:30,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/3e5e226bca8e488da386874f36199ac0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/3e5e226bca8e488da386874f36199ac0 2024-11-08T02:24:30,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/947c31d4da3144f783113d195e4ca5e8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/947c31d4da3144f783113d195e4ca5e8 2024-11-08T02:24:30,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/665208f498e14360841df71115744599 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/665208f498e14360841df71115744599 2024-11-08T02:24:30,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/054f6a6992204943822cc8793aa52d03 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/054f6a6992204943822cc8793aa52d03 2024-11-08T02:24:30,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5ad41fd9fe624c04b6571486ff8e9d41 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5ad41fd9fe624c04b6571486ff8e9d41 2024-11-08T02:24:30,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/6c0f1cc373564d7abc74823546a7f63e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/6c0f1cc373564d7abc74823546a7f63e 2024-11-08T02:24:30,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5da2ffa88738456889035c4aa73e324d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/5da2ffa88738456889035c4aa73e324d 2024-11-08T02:24:30,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/84098ec435ee49adb242d010c5f8297f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/84098ec435ee49adb242d010c5f8297f 2024-11-08T02:24:30,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b3f70c0b0d5f4464b789f567aa982021 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/b3f70c0b0d5f4464b789f567aa982021 2024-11-08T02:24:30,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/da212e0278904827ac30a0130b21d200 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/da212e0278904827ac30a0130b21d200 2024-11-08T02:24:30,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/74dc4593b87c4bb7b370516b5c7e5c34 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/74dc4593b87c4bb7b370516b5c7e5c34 2024-11-08T02:24:30,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/4fc00f03244243c99014fb8bc65506d2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/ecf72282e1504bc3b8e451fff4b35f1d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/332b57db6ddc4f9788584b5dfa73acda, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/3f19595d92a84c8b9279706a819b98f3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/df0591f03b2e4e36aa6eacda2419f219, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/328afc932d4645cbb6300cb58cc30793, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/f47d00c6579740a98ba6c9ca14d7020e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/bb3bd0b7b2e04aa0a8c043961cd922a8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6eb9fece42394040b9d57b4392bbe44d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/85976ccb556048919da31321cdbd5fcf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/906f5f93102648ed993e29ff374fa779, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/5184edd8fe274d8e97888743ca501497, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/15ad8817d38f4c01867f4fef078a74b0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/0c0dedfb48094b64bd67a56f2a5f1855, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/8ec6afc29857464baf214ea936a482e2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/06d3082b187e49c892668f2133ab9106, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/07a8a472987845a09fde2782261be599, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/1b55ac6fb2bc4c4fa54708c5e5420c9a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/9dd8da676fed44889f6280ee9f52645c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6435d3a4959740b491b3e953a28703d2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/56d461a272494d60828b1849477dab9d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/9fc3c346e325499a88b63b39ba23d48b] to archive 2024-11-08T02:24:30,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:24:30,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/4fc00f03244243c99014fb8bc65506d2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/4fc00f03244243c99014fb8bc65506d2 2024-11-08T02:24:30,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/ecf72282e1504bc3b8e451fff4b35f1d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/ecf72282e1504bc3b8e451fff4b35f1d 2024-11-08T02:24:30,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/332b57db6ddc4f9788584b5dfa73acda to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/332b57db6ddc4f9788584b5dfa73acda 2024-11-08T02:24:30,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/3f19595d92a84c8b9279706a819b98f3 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/3f19595d92a84c8b9279706a819b98f3 2024-11-08T02:24:30,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/df0591f03b2e4e36aa6eacda2419f219 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/df0591f03b2e4e36aa6eacda2419f219 2024-11-08T02:24:30,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/328afc932d4645cbb6300cb58cc30793 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/328afc932d4645cbb6300cb58cc30793 2024-11-08T02:24:30,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/f47d00c6579740a98ba6c9ca14d7020e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/f47d00c6579740a98ba6c9ca14d7020e 2024-11-08T02:24:30,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/bb3bd0b7b2e04aa0a8c043961cd922a8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/bb3bd0b7b2e04aa0a8c043961cd922a8 2024-11-08T02:24:30,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6eb9fece42394040b9d57b4392bbe44d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6eb9fece42394040b9d57b4392bbe44d 2024-11-08T02:24:30,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/85976ccb556048919da31321cdbd5fcf to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/85976ccb556048919da31321cdbd5fcf 2024-11-08T02:24:30,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/906f5f93102648ed993e29ff374fa779 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/906f5f93102648ed993e29ff374fa779 2024-11-08T02:24:30,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/5184edd8fe274d8e97888743ca501497 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/5184edd8fe274d8e97888743ca501497 2024-11-08T02:24:30,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/15ad8817d38f4c01867f4fef078a74b0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/15ad8817d38f4c01867f4fef078a74b0 2024-11-08T02:24:30,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/0c0dedfb48094b64bd67a56f2a5f1855 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/0c0dedfb48094b64bd67a56f2a5f1855 2024-11-08T02:24:30,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/8ec6afc29857464baf214ea936a482e2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/8ec6afc29857464baf214ea936a482e2 2024-11-08T02:24:30,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/06d3082b187e49c892668f2133ab9106 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/06d3082b187e49c892668f2133ab9106 2024-11-08T02:24:30,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/07a8a472987845a09fde2782261be599 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/07a8a472987845a09fde2782261be599 2024-11-08T02:24:30,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/1b55ac6fb2bc4c4fa54708c5e5420c9a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/1b55ac6fb2bc4c4fa54708c5e5420c9a 2024-11-08T02:24:30,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/9dd8da676fed44889f6280ee9f52645c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/9dd8da676fed44889f6280ee9f52645c 2024-11-08T02:24:30,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6435d3a4959740b491b3e953a28703d2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/6435d3a4959740b491b3e953a28703d2 2024-11-08T02:24:30,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/56d461a272494d60828b1849477dab9d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/56d461a272494d60828b1849477dab9d 2024-11-08T02:24:30,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/9fc3c346e325499a88b63b39ba23d48b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/9fc3c346e325499a88b63b39ba23d48b 2024-11-08T02:24:30,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/c5587019912e4f539cd038706cfc0d36, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f4c62a50b2334b578f6b4617fae1f7e8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/cc8a91776e744bb7a73e9d1f27f7233d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/8de7af73433a422091ecd87992f3ab58, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/2e1272ab6dd548f8b4b7e69033d0d0cf, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/ff6e1142148945f5b2b7a98de11c8dc0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6025b371c9dd4a3aad743f4891ee1bbe, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/93075fe379124dffb321b6fa99e162af, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/1623cf7d12164e1e9dd00c5de98d3db1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/c4f1a5fa410f401e919215179ac26dd0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/71ce85087b3f43818a1925d383a6e367, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/5e81adad23004a848f93a486df98cf95, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/0af0a13d8d9843a2a7c96fab4ebb66a2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/d12172bfaa354d1683baca6a5940b9d0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/e2e6d5fb4ec44bf49c5cc899e7b0683a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/a2c454e0e4de4436a4503dc4261da877, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6c5bdc5e7a23436d870090a59ba79515, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f3e94e1b956d49bf8b7a857d3fd6fce0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6ec70a3b50264cf7bc752be3cd456a14, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f6a9cde4a9a145149d833a4e3eea3729, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f80a94640a774bc0bd12364256fb1330, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/a6992e2dab054c62b3f91f43c8365ff6] to archive 2024-11-08T02:24:30,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:24:30,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/c5587019912e4f539cd038706cfc0d36 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/c5587019912e4f539cd038706cfc0d36 2024-11-08T02:24:30,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f4c62a50b2334b578f6b4617fae1f7e8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f4c62a50b2334b578f6b4617fae1f7e8 2024-11-08T02:24:30,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/cc8a91776e744bb7a73e9d1f27f7233d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/cc8a91776e744bb7a73e9d1f27f7233d 2024-11-08T02:24:30,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/8de7af73433a422091ecd87992f3ab58 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/8de7af73433a422091ecd87992f3ab58 2024-11-08T02:24:30,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/2e1272ab6dd548f8b4b7e69033d0d0cf to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/2e1272ab6dd548f8b4b7e69033d0d0cf 2024-11-08T02:24:30,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/ff6e1142148945f5b2b7a98de11c8dc0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/ff6e1142148945f5b2b7a98de11c8dc0 2024-11-08T02:24:30,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6025b371c9dd4a3aad743f4891ee1bbe to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6025b371c9dd4a3aad743f4891ee1bbe 2024-11-08T02:24:30,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/93075fe379124dffb321b6fa99e162af to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/93075fe379124dffb321b6fa99e162af 2024-11-08T02:24:30,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/1623cf7d12164e1e9dd00c5de98d3db1 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/1623cf7d12164e1e9dd00c5de98d3db1 2024-11-08T02:24:30,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/c4f1a5fa410f401e919215179ac26dd0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/c4f1a5fa410f401e919215179ac26dd0 2024-11-08T02:24:30,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/71ce85087b3f43818a1925d383a6e367 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/71ce85087b3f43818a1925d383a6e367 2024-11-08T02:24:30,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/5e81adad23004a848f93a486df98cf95 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/5e81adad23004a848f93a486df98cf95 2024-11-08T02:24:30,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/0af0a13d8d9843a2a7c96fab4ebb66a2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/0af0a13d8d9843a2a7c96fab4ebb66a2 2024-11-08T02:24:30,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/d12172bfaa354d1683baca6a5940b9d0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/d12172bfaa354d1683baca6a5940b9d0 2024-11-08T02:24:30,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/e2e6d5fb4ec44bf49c5cc899e7b0683a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/e2e6d5fb4ec44bf49c5cc899e7b0683a 2024-11-08T02:24:30,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/a2c454e0e4de4436a4503dc4261da877 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/a2c454e0e4de4436a4503dc4261da877 2024-11-08T02:24:30,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6c5bdc5e7a23436d870090a59ba79515 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6c5bdc5e7a23436d870090a59ba79515 2024-11-08T02:24:30,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f3e94e1b956d49bf8b7a857d3fd6fce0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f3e94e1b956d49bf8b7a857d3fd6fce0 2024-11-08T02:24:30,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6ec70a3b50264cf7bc752be3cd456a14 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6ec70a3b50264cf7bc752be3cd456a14 2024-11-08T02:24:30,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f6a9cde4a9a145149d833a4e3eea3729 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f6a9cde4a9a145149d833a4e3eea3729 2024-11-08T02:24:30,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f80a94640a774bc0bd12364256fb1330 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/f80a94640a774bc0bd12364256fb1330 2024-11-08T02:24:30,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/a6992e2dab054c62b3f91f43c8365ff6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/a6992e2dab054c62b3f91f43c8365ff6 2024-11-08T02:24:30,436 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/recovered.edits/348.seqid, newMaxSeqId=348, maxSeqId=4 2024-11-08T02:24:30,436 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. 2024-11-08T02:24:30,436 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for a03465591aab9088b0f79bf5ef3a6e88: Waiting for close lock at 1731032668458Running coprocessor pre-close hooks at 1731032668458Disabling compacts and flushes for region at 1731032668458Disabling writes for close at 1731032668627 (+169 ms)Obtaining lock to block concurrent updates at 1731032668628 (+1 ms)Preparing flush snapshotting stores in a03465591aab9088b0f79bf5ef3a6e88 at 1731032668628Finished memstore snapshotting TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731032668628Flushing stores of TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88. at 1731032668628Flushing a03465591aab9088b0f79bf5ef3a6e88/A: creating writer at 1731032668628Flushing mob file a03465591aab9088b0f79bf5ef3a6e88/A: appending metadata at 1731032668634 (+6 ms)Flushing mob file a03465591aab9088b0f79bf5ef3a6e88/A: closing flushed file at 1731032668634Flushing a03465591aab9088b0f79bf5ef3a6e88/A: appending metadata at 1731032669050 (+416 ms)Flushing a03465591aab9088b0f79bf5ef3a6e88/A: closing flushed file at 1731032669050Flushing a03465591aab9088b0f79bf5ef3a6e88/B: creating writer at 1731032669465 (+415 ms)Flushing a03465591aab9088b0f79bf5ef3a6e88/B: appending metadata at 1731032669468 (+3 ms)Flushing a03465591aab9088b0f79bf5ef3a6e88/B: closing flushed file at 1731032669468Flushing a03465591aab9088b0f79bf5ef3a6e88/C: creating writer at 1731032669883 (+415 ms)Flushing a03465591aab9088b0f79bf5ef3a6e88/C: appending metadata at 1731032669887 (+4 ms)Flushing a03465591aab9088b0f79bf5ef3a6e88/C: closing flushed file at 1731032669888 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b8d43d7: reopening flushed file at 1731032670306 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bbfeff5: reopening flushed file at 1731032670315 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f67b704: reopening flushed file at 1731032670322 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a03465591aab9088b0f79bf5ef3a6e88 in 1700ms, sequenceid=345, compaction requested=true at 1731032670328 (+6 ms)Writing region close event to WAL at 1731032670432 (+104 ms)Running coprocessor post-close hooks at 1731032670436 (+4 ms)Closed at 1731032670436 2024-11-08T02:24:30,439 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:30,439 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=a03465591aab9088b0f79bf5ef3a6e88, regionState=CLOSED 2024-11-08T02:24:30,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:24:30,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-08T02:24:30,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure a03465591aab9088b0f79bf5ef3a6e88, server=331c7316141f,34325,1731032548147 in 2.1370 sec 2024-11-08T02:24:30,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-08T02:24:30,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03465591aab9088b0f79bf5ef3a6e88, UNASSIGN in 2.1420 sec 2024-11-08T02:24:30,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-08T02:24:30,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1450 sec 2024-11-08T02:24:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T02:24:30,449 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032670449"}]},"ts":"1731032670449"} 2024-11-08T02:24:30,450 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:24:30,450 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:24:30,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1550 sec 2024-11-08T02:24:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T02:24:32,469 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:32,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:24:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:32,475 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T02:24:32,477 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:32,480 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,482 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/recovered.edits] 2024-11-08T02:24:32,484 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/20e3932cc7cb4374925714206f85e770 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/20e3932cc7cb4374925714206f85e770 2024-11-08T02:24:32,485 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/71b665ccecf340abb658de9269cff5fe to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/71b665ccecf340abb658de9269cff5fe 2024-11-08T02:24:32,485 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/a70daf3c2afb422b960420ee7b7490a0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/a70daf3c2afb422b960420ee7b7490a0 2024-11-08T02:24:32,486 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/f916a7f3daec4817a7aaf5fb1ded1d9d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/A/f916a7f3daec4817a7aaf5fb1ded1d9d 2024-11-08T02:24:32,488 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/1a4d842198f04449a67291b4f7423736 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/1a4d842198f04449a67291b4f7423736 2024-11-08T02:24:32,489 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/429d601b983b453fa7030c7803424b72 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/429d601b983b453fa7030c7803424b72 2024-11-08T02:24:32,490 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/ea43515a6b4246e583971b9b54939a45 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/ea43515a6b4246e583971b9b54939a45 2024-11-08T02:24:32,490 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/f9e8d8aa4a6f4f08951ecf0ce5566036 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/B/f9e8d8aa4a6f4f08951ecf0ce5566036 2024-11-08T02:24:32,492 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/1a4b1b8c35594ab6a382b886529d5bba to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/1a4b1b8c35594ab6a382b886529d5bba 2024-11-08T02:24:32,493 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6390a5bc134f41a39c2ab304c80bbda3 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/6390a5bc134f41a39c2ab304c80bbda3 2024-11-08T02:24:32,494 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/91f48a70a38745f68c6fe855d8882ddd to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/91f48a70a38745f68c6fe855d8882ddd 2024-11-08T02:24:32,494 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/e02189abf73940be93733e4bc6f96eac to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/C/e02189abf73940be93733e4bc6f96eac 2024-11-08T02:24:32,497 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/recovered.edits/348.seqid to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88/recovered.edits/348.seqid 2024-11-08T02:24:32,497 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,497 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:24:32,497 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:24:32,498 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T02:24:32,500 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110817edb8a003124a368588c9574c6ebf36_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110817edb8a003124a368588c9574c6ebf36_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,501 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110818947377c8d44619ad8bddc4187df785_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110818947377c8d44619ad8bddc4187df785_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,502 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110823cd7e12e2864534a99125be2a94fdfc_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110823cd7e12e2864534a99125be2a94fdfc_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,503 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082847bafa6f1f4799bb88a3cb8ff1f3d0_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082847bafa6f1f4799bb88a3cb8ff1f3d0_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,504 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082f57579d6b9f44689503689afd1ce4b9_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082f57579d6b9f44689503689afd1ce4b9_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,504 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108332259eaa51b490c93568b3c0ddb3b07_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108332259eaa51b490c93568b3c0ddb3b07_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,505 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110842888b2ae003496ba3be501010dd020e_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110842888b2ae003496ba3be501010dd020e_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,506 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110850170a30abb54fd4836335030156bfac_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110850170a30abb54fd4836335030156bfac_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,506 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086788c5487b114eb79607dc905480d46e_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086788c5487b114eb79607dc905480d46e_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,507 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108758ec9fabd8245859008a5bead4fccad_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108758ec9fabd8245859008a5bead4fccad_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,507 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110895332f38a63047fa800b8e51f23dfc7e_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110895332f38a63047fa800b8e51f23dfc7e_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,508 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ac37aaf482cf43dfbc75b12cab952788_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ac37aaf482cf43dfbc75b12cab952788_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,509 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bffd68d0d8ea4d52a91de48fcd681fac_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bffd68d0d8ea4d52a91de48fcd681fac_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,509 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c70eb3b1fd2b4ddd884814ede999923d_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c70eb3b1fd2b4ddd884814ede999923d_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,510 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d9335d6bd5534610b30fc4f2470394cc_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d9335d6bd5534610b30fc4f2470394cc_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,510 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108db31851ffa70423482ff58ed347bd3b2_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108db31851ffa70423482ff58ed347bd3b2_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,511 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108df7f2ee226a34d33a1c5b4d558d1eca8_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108df7f2ee226a34d33a1c5b4d558d1eca8_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,512 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e677de14aeba4b42b2a4945be5fe9536_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e677de14aeba4b42b2a4945be5fe9536_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,512 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108feef677c56524f17aa2c57acaa99b58e_a03465591aab9088b0f79bf5ef3a6e88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108feef677c56524f17aa2c57acaa99b58e_a03465591aab9088b0f79bf5ef3a6e88 2024-11-08T02:24:32,513 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:24:32,514 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:32,515 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:24:32,517 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:24:32,517 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:32,517 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:24:32,518 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731032672517"}]},"ts":"9223372036854775807"} 2024-11-08T02:24:32,519 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T02:24:32,519 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a03465591aab9088b0f79bf5ef3a6e88, NAME => 'TestAcidGuarantees,,1731032643155.a03465591aab9088b0f79bf5ef3a6e88.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:24:32,519 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:24:32,519 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731032672519"}]},"ts":"9223372036854775807"} 2024-11-08T02:24:32,520 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T02:24:32,521 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:32,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-08T02:24:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T02:24:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T02:24:32,588 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:32,596 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 240) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-403120256_22 at /127.0.0.1:49740 [Waiting for operation #2007] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-548309475_22 at /127.0.0.1:54520 [Waiting for operation #1607] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-403120256_22 at /127.0.0.1:40590 [Waiting for operation #1492] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/331c7316141f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-548309475_22 at /127.0.0.1:40588 [Waiting for operation #1511] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=512 (was 546), ProcessCount=11 (was 11), AvailableMemoryMB=6650 (was 6719) 2024-11-08T02:24:32,603 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=512, ProcessCount=11, AvailableMemoryMB=6650 2024-11-08T02:24:32,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:24:32,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:24:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:24:32,606 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:24:32,606 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:32,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-08T02:24:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-08T02:24:32,607 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:24:32,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742264_1440 (size=959) 2024-11-08T02:24:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-08T02:24:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-08T02:24:33,017 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee 2024-11-08T02:24:33,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742265_1441 (size=53) 2024-11-08T02:24:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-08T02:24:33,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:24:33,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9024be227c18091a068b12c5acebba9f, disabling compactions & flushes 2024-11-08T02:24:33,427 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:33,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:33,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. after waiting 0 ms 2024-11-08T02:24:33,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:33,427 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:33,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9024be227c18091a068b12c5acebba9f: Waiting for close lock at 1731032673427Disabling compacts and flushes for region at 1731032673427Disabling writes for close at 1731032673427Writing region close event to WAL at 1731032673427Closed at 1731032673427 2024-11-08T02:24:33,430 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:24:33,430 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731032673430"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731032673430"}]},"ts":"1731032673430"} 2024-11-08T02:24:33,433 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T02:24:33,434 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:24:33,435 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032673435"}]},"ts":"1731032673435"} 2024-11-08T02:24:33,437 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:24:33,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9024be227c18091a068b12c5acebba9f, ASSIGN}] 2024-11-08T02:24:33,438 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9024be227c18091a068b12c5acebba9f, ASSIGN 2024-11-08T02:24:33,439 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9024be227c18091a068b12c5acebba9f, ASSIGN; state=OFFLINE, location=331c7316141f,34325,1731032548147; forceNewPlan=false, retain=false 2024-11-08T02:24:33,590 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=9024be227c18091a068b12c5acebba9f, regionState=OPENING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:24:33,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9024be227c18091a068b12c5acebba9f, ASSIGN because future has completed 2024-11-08T02:24:33,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:24:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-08T02:24:33,756 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:33,757 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:24:33,758 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,758 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:24:33,758 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,758 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,760 INFO [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,762 INFO [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9024be227c18091a068b12c5acebba9f columnFamilyName A 2024-11-08T02:24:33,762 DEBUG [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:33,763 INFO [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] regionserver.HStore(327): Store=9024be227c18091a068b12c5acebba9f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:24:33,763 INFO [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,765 INFO [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9024be227c18091a068b12c5acebba9f columnFamilyName B 2024-11-08T02:24:33,765 DEBUG [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:33,765 INFO [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] regionserver.HStore(327): Store=9024be227c18091a068b12c5acebba9f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:24:33,766 INFO [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,767 INFO [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9024be227c18091a068b12c5acebba9f columnFamilyName C 2024-11-08T02:24:33,767 DEBUG [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:24:33,768 INFO [StoreOpener-9024be227c18091a068b12c5acebba9f-1 {}] regionserver.HStore(327): Store=9024be227c18091a068b12c5acebba9f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:24:33,768 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,769 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,770 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,771 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,771 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,772 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:24:33,773 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,775 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:24:33,775 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 9024be227c18091a068b12c5acebba9f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71323980, jitterRate=0.06281012296676636}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:24:33,775 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:33,776 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 9024be227c18091a068b12c5acebba9f: Running coprocessor pre-open hook at 1731032673758Writing region info on filesystem at 1731032673758Initializing all the Stores at 1731032673760 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032673760Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032673760Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032673760Cleaning up temporary data from old regions at 1731032673771 (+11 ms)Running coprocessor post-open hooks at 1731032673775 (+4 ms)Region opened successfully at 1731032673776 (+1 ms) 2024-11-08T02:24:33,777 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., pid=118, masterSystemTime=1731032673748 2024-11-08T02:24:33,778 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:33,778 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:33,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=9024be227c18091a068b12c5acebba9f, regionState=OPEN, openSeqNum=2, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:24:33,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:24:33,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-08T02:24:33,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 in 187 msec 2024-11-08T02:24:33,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-08T02:24:33,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9024be227c18091a068b12c5acebba9f, ASSIGN in 345 msec 2024-11-08T02:24:33,783 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:24:33,784 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032673783"}]},"ts":"1731032673783"} 2024-11-08T02:24:33,785 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:24:33,785 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:24:33,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-08T02:24:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-08T02:24:34,759 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:34,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:34,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:34,762 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@718be1a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:34,765 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb4779a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,766 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:34,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:34,770 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T02:24:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 331c7316141f,41605,1731032547397 2024-11-08T02:24:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36d9e655 2024-11-08T02:24:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:24:34,772 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:24:34,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:34,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:34,775 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40eaa9d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:34,777 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:34,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1623c4ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:34,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:34,779 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:34,781 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb187bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:34,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:34,784 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fdb748, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:34,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:34,785 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2905c34d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,786 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:34,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:34,787 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746e8f8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:34,789 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b6c866b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,790 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:34,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:34,791 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:34,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:34,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:34,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77cfaec3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:34,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:34,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:34,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f11595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:34,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:34,795 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46604091, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:34,796 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:34,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f1b054a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:34,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:34,799 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14378b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:34,801 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3deb970d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:34,802 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:34,803 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@766f05a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:34,804 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af75af6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:34,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:34,806 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b546bef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:34,808 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0d5c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,809 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:24:34,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:24:34,810 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:24:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:24:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:24:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b052f8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:24:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:24:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:34,811 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:24:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70206d7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:24:34,812 DEBUG [Thread-1983 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:34,812 DEBUG [Thread-1996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:34,812 DEBUG [Thread-1992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:34,812 DEBUG [Thread-1981 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:34,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:34,813 DEBUG [Thread-2000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:34,813 DEBUG [Thread-1992 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:34,813 DEBUG [Thread-1992 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:34,814 DEBUG [Thread-1987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:34,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:34,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-08T02:24:34,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:34,815 DEBUG [Thread-1989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:34,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T02:24:34,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:24:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:24:34,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:34,817 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:34,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:34,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:34,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:24:34,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:24:34,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:24:34,816 DEBUG [Thread-1989 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:34,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:34,819 DEBUG [Thread-1989 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:34,819 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:34,819 DEBUG [Thread-1998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:34,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:24:34,820 DEBUG [Thread-1998 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:34,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:34,821 DEBUG [Thread-1985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:34,821 DEBUG [Thread-1994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:24:34,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:24:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:24:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:24:34,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:24:34,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:34,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:24:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:24:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2] 2024-11-08T02:24:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:34,831 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:34,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:24:34,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/77fed8fd945c4684a36da477c51de08e is 50, key is test_row_0/A:col10/1731032674825/Put/seqid=0 2024-11-08T02:24:34,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742266_1442 (size=12001) 2024-11-08T02:24:34,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032734868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:34,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032734869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:34,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032734876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:34,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032734877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032734878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T02:24:34,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T02:24:34,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:34,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:34,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:34,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:34,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:34,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032734988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:34,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032734988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:34,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032734988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:34,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032734988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:34,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032734989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:35,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T02:24:35,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:35,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:35,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:35,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T02:24:35,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032735198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:35,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032735198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:35,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032735199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032735199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:35,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032735208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:35,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/77fed8fd945c4684a36da477c51de08e 2024-11-08T02:24:35,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/23bd6c1e4ab24503976ff169ecdff32a is 50, key is test_row_0/B:col10/1731032674825/Put/seqid=0 2024-11-08T02:24:35,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742267_1443 (size=12001) 2024-11-08T02:24:35,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T02:24:35,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:35,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:35,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:35,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T02:24:35,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:35,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:35,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:35,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T02:24:35,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032735508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:35,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:35,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032735508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:35,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032735508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:35,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032735509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:35,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032735518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:35,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T02:24:35,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:35,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:35,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:35,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:35,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/23bd6c1e4ab24503976ff169ecdff32a 2024-11-08T02:24:35,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/d9806859a91f45bb8332b33944877576 is 50, key is test_row_0/C:col10/1731032674825/Put/seqid=0 2024-11-08T02:24:35,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742268_1444 (size=12001) 2024-11-08T02:24:35,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/d9806859a91f45bb8332b33944877576 2024-11-08T02:24:35,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/77fed8fd945c4684a36da477c51de08e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/77fed8fd945c4684a36da477c51de08e 2024-11-08T02:24:35,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/77fed8fd945c4684a36da477c51de08e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T02:24:35,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/23bd6c1e4ab24503976ff169ecdff32a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/23bd6c1e4ab24503976ff169ecdff32a 2024-11-08T02:24:35,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/23bd6c1e4ab24503976ff169ecdff32a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T02:24:35,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/d9806859a91f45bb8332b33944877576 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/d9806859a91f45bb8332b33944877576 2024-11-08T02:24:35,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/d9806859a91f45bb8332b33944877576, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T02:24:35,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 9024be227c18091a068b12c5acebba9f in 889ms, sequenceid=12, compaction requested=false 2024-11-08T02:24:35,720 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T02:24:35,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:35,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T02:24:35,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:35,737 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:24:35,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/11ce8f180da4495cb0cec22c357282c5 is 50, key is test_row_0/A:col10/1731032674876/Put/seqid=0 2024-11-08T02:24:35,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742269_1445 (size=12001) 2024-11-08T02:24:35,953 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:24:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T02:24:36,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:36,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032736028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032736028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032736031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032736032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032736033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,144 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/11ce8f180da4495cb0cec22c357282c5 2024-11-08T02:24:36,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/696abaeac09246aebb81442ac210148d is 50, key is test_row_0/B:col10/1731032674876/Put/seqid=0 2024-11-08T02:24:36,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032736148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032736148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032736148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742270_1446 (size=12001) 2024-11-08T02:24:36,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032736358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032736358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032736358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,553 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/696abaeac09246aebb81442ac210148d 2024-11-08T02:24:36,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/b4f1db6e035349e884c528179a5da868 is 50, key is test_row_0/C:col10/1731032674876/Put/seqid=0 2024-11-08T02:24:36,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742271_1447 (size=12001) 2024-11-08T02:24:36,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032736667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032736667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032736667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T02:24:37,008 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/b4f1db6e035349e884c528179a5da868 2024-11-08T02:24:37,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/11ce8f180da4495cb0cec22c357282c5 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/11ce8f180da4495cb0cec22c357282c5 2024-11-08T02:24:37,014 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/11ce8f180da4495cb0cec22c357282c5, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T02:24:37,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/696abaeac09246aebb81442ac210148d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/696abaeac09246aebb81442ac210148d 2024-11-08T02:24:37,018 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/696abaeac09246aebb81442ac210148d, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T02:24:37,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/b4f1db6e035349e884c528179a5da868 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/b4f1db6e035349e884c528179a5da868 2024-11-08T02:24:37,021 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/b4f1db6e035349e884c528179a5da868, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T02:24:37,022 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 9024be227c18091a068b12c5acebba9f in 1285ms, sequenceid=38, compaction requested=false 2024-11-08T02:24:37,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:37,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:37,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-08T02:24:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-08T02:24:37,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-08T02:24:37,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2040 sec 2024-11-08T02:24:37,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.2120 sec 2024-11-08T02:24:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:37,052 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:37,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:24:37,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/6d3f234c9ffb4bbb8c48e31e85395d7c is 50, key is test_row_0/A:col10/1731032676030/Put/seqid=0 2024-11-08T02:24:37,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742272_1448 (size=14341) 2024-11-08T02:24:37,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032737133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:37,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032737134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:37,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032737178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032737178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:37,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032737188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:37,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032737238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:37,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:37,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032737248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:37,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032737447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:37,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:37,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/6d3f234c9ffb4bbb8c48e31e85395d7c 2024-11-08T02:24:37,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/8067695e3c5b4376832b6a1ab6c477e1 is 50, key is test_row_0/B:col10/1731032676030/Put/seqid=0 2024-11-08T02:24:37,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742273_1449 (size=12001) 2024-11-08T02:24:37,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032737457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:37,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032737768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:37,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032737778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:37,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:37,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/8067695e3c5b4376832b6a1ab6c477e1 2024-11-08T02:24:37,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c4cdd0c9a7904957aec6aa39ce31001a is 50, key is test_row_0/C:col10/1731032676030/Put/seqid=0 2024-11-08T02:24:37,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742274_1450 (size=12001) 2024-11-08T02:24:38,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032738197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:38,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032738197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:38,202 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:38,202 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:38,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032738208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:38,211 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:38,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c4cdd0c9a7904957aec6aa39ce31001a 2024-11-08T02:24:38,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/6d3f234c9ffb4bbb8c48e31e85395d7c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/6d3f234c9ffb4bbb8c48e31e85395d7c 2024-11-08T02:24:38,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/6d3f234c9ffb4bbb8c48e31e85395d7c, entries=200, sequenceid=49, filesize=14.0 K 2024-11-08T02:24:38,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/8067695e3c5b4376832b6a1ab6c477e1 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/8067695e3c5b4376832b6a1ab6c477e1 2024-11-08T02:24:38,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/8067695e3c5b4376832b6a1ab6c477e1, entries=150, sequenceid=49, filesize=11.7 K 2024-11-08T02:24:38,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c4cdd0c9a7904957aec6aa39ce31001a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c4cdd0c9a7904957aec6aa39ce31001a 2024-11-08T02:24:38,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032738288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:38,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c4cdd0c9a7904957aec6aa39ce31001a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-08T02:24:38,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9024be227c18091a068b12c5acebba9f in 1248ms, sequenceid=49, compaction requested=true 2024-11-08T02:24:38,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:38,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:38,301 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:38,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:38,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:38,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:38,301 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:38,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:38,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:38,302 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:38,302 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/B is initiating minor compaction (all files) 2024-11-08T02:24:38,302 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/B in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:38,302 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/23bd6c1e4ab24503976ff169ecdff32a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/696abaeac09246aebb81442ac210148d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/8067695e3c5b4376832b6a1ab6c477e1] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=35.2 K 2024-11-08T02:24:38,302 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:38,302 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/A is initiating minor compaction (all files) 2024-11-08T02:24:38,302 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/A in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:38,303 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/77fed8fd945c4684a36da477c51de08e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/11ce8f180da4495cb0cec22c357282c5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/6d3f234c9ffb4bbb8c48e31e85395d7c] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=37.4 K 2024-11-08T02:24:38,303 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 23bd6c1e4ab24503976ff169ecdff32a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731032674824 2024-11-08T02:24:38,303 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77fed8fd945c4684a36da477c51de08e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731032674824 2024-11-08T02:24:38,303 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 696abaeac09246aebb81442ac210148d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731032674868 2024-11-08T02:24:38,303 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11ce8f180da4495cb0cec22c357282c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731032674868 2024-11-08T02:24:38,303 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 8067695e3c5b4376832b6a1ab6c477e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731032676030 2024-11-08T02:24:38,304 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d3f234c9ffb4bbb8c48e31e85395d7c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731032676023 2024-11-08T02:24:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:38,309 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:38,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:24:38,313 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#A#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:38,313 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:38,314 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/ed4c3b3f20df469d969f4a9a6a17a6f3 is 50, key is test_row_0/A:col10/1731032676030/Put/seqid=0 2024-11-08T02:24:38,314 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/9ea19cc57a0147eb9410d23c6f8257ba is 50, key is test_row_0/B:col10/1731032676030/Put/seqid=0 2024-11-08T02:24:38,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/1bd28cc941954ccc8df85a607f0ce44e is 50, key is test_row_0/A:col10/1731032677125/Put/seqid=0 2024-11-08T02:24:38,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742275_1451 (size=12104) 2024-11-08T02:24:38,349 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/ed4c3b3f20df469d969f4a9a6a17a6f3 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/ed4c3b3f20df469d969f4a9a6a17a6f3 2024-11-08T02:24:38,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742277_1453 (size=14341) 2024-11-08T02:24:38,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742276_1452 (size=12104) 2024-11-08T02:24:38,354 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/A of 9024be227c18091a068b12c5acebba9f into ed4c3b3f20df469d969f4a9a6a17a6f3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:38,354 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:38,354 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/A, priority=13, startTime=1731032678301; duration=0sec 2024-11-08T02:24:38,354 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:38,354 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:A 2024-11-08T02:24:38,355 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:38,355 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:38,355 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/C is initiating minor compaction (all files) 2024-11-08T02:24:38,355 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/C in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:38,356 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/d9806859a91f45bb8332b33944877576, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/b4f1db6e035349e884c528179a5da868, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c4cdd0c9a7904957aec6aa39ce31001a] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=35.2 K 2024-11-08T02:24:38,356 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9806859a91f45bb8332b33944877576, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731032674824 2024-11-08T02:24:38,356 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4f1db6e035349e884c528179a5da868, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731032674868 2024-11-08T02:24:38,356 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4cdd0c9a7904957aec6aa39ce31001a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731032676030 2024-11-08T02:24:38,362 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:38,363 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/bb9d55ca77ec4ee59d55ab0d4971b328 is 50, key is test_row_0/C:col10/1731032676030/Put/seqid=0 2024-11-08T02:24:38,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742278_1454 (size=12104) 2024-11-08T02:24:38,373 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/bb9d55ca77ec4ee59d55ab0d4971b328 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/bb9d55ca77ec4ee59d55ab0d4971b328 2024-11-08T02:24:38,376 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/C of 9024be227c18091a068b12c5acebba9f into bb9d55ca77ec4ee59d55ab0d4971b328(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:38,376 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:38,376 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/C, priority=13, startTime=1731032678301; duration=0sec 2024-11-08T02:24:38,376 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:38,376 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:C 2024-11-08T02:24:38,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032738368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:38,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032738488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:38,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032738698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:38,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:38,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/1bd28cc941954ccc8df85a607f0ce44e 2024-11-08T02:24:38,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/89555b4ef3c8481e98e0e1dd8b996f77 is 50, key is test_row_0/B:col10/1731032677125/Put/seqid=0 2024-11-08T02:24:38,757 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/9ea19cc57a0147eb9410d23c6f8257ba as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/9ea19cc57a0147eb9410d23c6f8257ba 2024-11-08T02:24:38,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742279_1455 (size=12001) 2024-11-08T02:24:38,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/89555b4ef3c8481e98e0e1dd8b996f77 2024-11-08T02:24:38,762 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/B of 9024be227c18091a068b12c5acebba9f into 9ea19cc57a0147eb9410d23c6f8257ba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:38,762 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:38,762 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/B, priority=13, startTime=1731032678301; duration=0sec 2024-11-08T02:24:38,762 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:38,762 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:B 2024-11-08T02:24:38,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c83fece32e364ed79d9b30661c63fcc0 is 50, key is test_row_0/C:col10/1731032677125/Put/seqid=0 2024-11-08T02:24:38,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742280_1456 (size=12001) 2024-11-08T02:24:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T02:24:38,978 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:38,978 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-08T02:24:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T02:24:38,980 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:38,980 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:38,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:39,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032739008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:39,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T02:24:39,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T02:24:39,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:39,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:39,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:39,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:39,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:39,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c83fece32e364ed79d9b30661c63fcc0 2024-11-08T02:24:39,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/1bd28cc941954ccc8df85a607f0ce44e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/1bd28cc941954ccc8df85a607f0ce44e 2024-11-08T02:24:39,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/1bd28cc941954ccc8df85a607f0ce44e, entries=200, sequenceid=75, filesize=14.0 K 2024-11-08T02:24:39,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/89555b4ef3c8481e98e0e1dd8b996f77 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/89555b4ef3c8481e98e0e1dd8b996f77 2024-11-08T02:24:39,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/89555b4ef3c8481e98e0e1dd8b996f77, entries=150, sequenceid=75, filesize=11.7 K 2024-11-08T02:24:39,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c83fece32e364ed79d9b30661c63fcc0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c83fece32e364ed79d9b30661c63fcc0 2024-11-08T02:24:39,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c83fece32e364ed79d9b30661c63fcc0, entries=150, sequenceid=75, filesize=11.7 K 2024-11-08T02:24:39,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 9024be227c18091a068b12c5acebba9f in 875ms, sequenceid=75, compaction requested=false 2024-11-08T02:24:39,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:39,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T02:24:39,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:39,286 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T02:24:39,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/63a10216b71946818045a9fdce7e558a is 50, key is test_row_0/A:col10/1731032678360/Put/seqid=0 2024-11-08T02:24:39,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742281_1457 (size=12001) 2024-11-08T02:24:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T02:24:39,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:39,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032739451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:39,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032739517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032739558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:39,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T02:24:39,695 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/63a10216b71946818045a9fdce7e558a 2024-11-08T02:24:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/1aa5bf460a7941dc8799eba0907ab284 is 50, key is test_row_0/B:col10/1731032678360/Put/seqid=0 2024-11-08T02:24:39,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742282_1458 (size=12001) 2024-11-08T02:24:39,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032739768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032740078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,105 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/1aa5bf460a7941dc8799eba0907ab284 2024-11-08T02:24:40,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/2ce696d3cb664845a2dd722ade88738c is 50, key is test_row_0/C:col10/1731032678360/Put/seqid=0 2024-11-08T02:24:40,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742283_1459 (size=12001) 2024-11-08T02:24:40,114 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/2ce696d3cb664845a2dd722ade88738c 2024-11-08T02:24:40,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/63a10216b71946818045a9fdce7e558a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/63a10216b71946818045a9fdce7e558a 2024-11-08T02:24:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T02:24:40,121 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/63a10216b71946818045a9fdce7e558a, entries=150, sequenceid=88, filesize=11.7 K 2024-11-08T02:24:40,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/1aa5bf460a7941dc8799eba0907ab284 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/1aa5bf460a7941dc8799eba0907ab284 2024-11-08T02:24:40,124 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/1aa5bf460a7941dc8799eba0907ab284, entries=150, sequenceid=88, filesize=11.7 K 2024-11-08T02:24:40,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/2ce696d3cb664845a2dd722ade88738c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2ce696d3cb664845a2dd722ade88738c 2024-11-08T02:24:40,127 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2ce696d3cb664845a2dd722ade88738c, entries=150, sequenceid=88, filesize=11.7 K 2024-11-08T02:24:40,128 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 9024be227c18091a068b12c5acebba9f in 843ms, sequenceid=88, compaction requested=true 2024-11-08T02:24:40,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:40,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:40,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-08T02:24:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-08T02:24:40,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-08T02:24:40,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1490 sec 2024-11-08T02:24:40,132 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.1530 sec 2024-11-08T02:24:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:40,213 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:40,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T02:24:40,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/81ef2e7542104904bfd1c57b9f9f6f7d is 50, key is test_row_0/A:col10/1731032680211/Put/seqid=0 2024-11-08T02:24:40,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742284_1460 (size=14341) 2024-11-08T02:24:40,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032740233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032740236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032740237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032740348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032740348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032740348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032740537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,542 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:40,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032740558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032740558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032740558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032740587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/81ef2e7542104904bfd1c57b9f9f6f7d 2024-11-08T02:24:40,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/7627f957b9bf4ce99b6ff936fbcab975 is 50, key is test_row_0/B:col10/1731032680211/Put/seqid=0 2024-11-08T02:24:40,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742285_1461 (size=12001) 2024-11-08T02:24:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032740867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032740867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032740867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:41,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/7627f957b9bf4ce99b6ff936fbcab975 2024-11-08T02:24:41,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/a1bd551e2c6c44d187176a42cdda50c2 is 50, key is test_row_0/C:col10/1731032680211/Put/seqid=0 2024-11-08T02:24:41,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742286_1462 (size=12001) 2024-11-08T02:24:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T02:24:41,128 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:41,128 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-08T02:24:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T02:24:41,130 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:41,130 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:41,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T02:24:41,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T02:24:41,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:41,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:41,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:41,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:41,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032741378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032741378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:41,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:41,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032741378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:41,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T02:24:41,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:41,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:41,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:41,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:41,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:41,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/a1bd551e2c6c44d187176a42cdda50c2 2024-11-08T02:24:41,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/81ef2e7542104904bfd1c57b9f9f6f7d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/81ef2e7542104904bfd1c57b9f9f6f7d 2024-11-08T02:24:41,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/81ef2e7542104904bfd1c57b9f9f6f7d, entries=200, sequenceid=115, filesize=14.0 K 2024-11-08T02:24:41,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/7627f957b9bf4ce99b6ff936fbcab975 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/7627f957b9bf4ce99b6ff936fbcab975 2024-11-08T02:24:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T02:24:41,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/7627f957b9bf4ce99b6ff936fbcab975, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:24:41,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/a1bd551e2c6c44d187176a42cdda50c2 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/a1bd551e2c6c44d187176a42cdda50c2 2024-11-08T02:24:41,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/a1bd551e2c6c44d187176a42cdda50c2, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:24:41,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 9024be227c18091a068b12c5acebba9f in 1239ms, sequenceid=115, compaction requested=true 2024-11-08T02:24:41,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:41,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:41,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:41,453 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:24:41,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:41,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:41,453 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:24:41,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:41,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:41,453 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:24:41,453 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:24:41,453 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/B is initiating minor compaction (all files) 2024-11-08T02:24:41,453 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/A is initiating minor compaction (all files) 2024-11-08T02:24:41,454 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/B in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:41,454 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/A in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:41,454 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/ed4c3b3f20df469d969f4a9a6a17a6f3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/1bd28cc941954ccc8df85a607f0ce44e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/63a10216b71946818045a9fdce7e558a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/81ef2e7542104904bfd1c57b9f9f6f7d] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=51.5 K 2024-11-08T02:24:41,454 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/9ea19cc57a0147eb9410d23c6f8257ba, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/89555b4ef3c8481e98e0e1dd8b996f77, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/1aa5bf460a7941dc8799eba0907ab284, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/7627f957b9bf4ce99b6ff936fbcab975] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=47.0 K 2024-11-08T02:24:41,454 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed4c3b3f20df469d969f4a9a6a17a6f3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731032676030 2024-11-08T02:24:41,454 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ea19cc57a0147eb9410d23c6f8257ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731032676030 2024-11-08T02:24:41,454 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bd28cc941954ccc8df85a607f0ce44e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731032677102 2024-11-08T02:24:41,454 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 89555b4ef3c8481e98e0e1dd8b996f77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731032677102 2024-11-08T02:24:41,454 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63a10216b71946818045a9fdce7e558a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731032678347 2024-11-08T02:24:41,454 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aa5bf460a7941dc8799eba0907ab284, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731032678347 2024-11-08T02:24:41,455 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 7627f957b9bf4ce99b6ff936fbcab975, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731032679440 2024-11-08T02:24:41,455 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ef2e7542104904bfd1c57b9f9f6f7d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731032679440 2024-11-08T02:24:41,465 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#A#compaction#391 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:41,465 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#B#compaction#390 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:41,466 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/ba0ef423fd664345af427ac8e4a45adb is 50, key is test_row_0/B:col10/1731032680211/Put/seqid=0 2024-11-08T02:24:41,466 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/62b92e61a18d4551b94af4741694877f is 50, key is test_row_0/A:col10/1731032680211/Put/seqid=0 2024-11-08T02:24:41,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742288_1464 (size=12241) 2024-11-08T02:24:41,477 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/62b92e61a18d4551b94af4741694877f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/62b92e61a18d4551b94af4741694877f 2024-11-08T02:24:41,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742287_1463 (size=12241) 2024-11-08T02:24:41,486 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9024be227c18091a068b12c5acebba9f/A of 9024be227c18091a068b12c5acebba9f into 62b92e61a18d4551b94af4741694877f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:41,486 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:41,486 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/A, priority=12, startTime=1731032681452; duration=0sec 2024-11-08T02:24:41,486 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:41,486 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:A 2024-11-08T02:24:41,486 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:24:41,487 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:24:41,487 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/C is initiating minor compaction (all files) 2024-11-08T02:24:41,487 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/C in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:41,487 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/bb9d55ca77ec4ee59d55ab0d4971b328, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c83fece32e364ed79d9b30661c63fcc0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2ce696d3cb664845a2dd722ade88738c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/a1bd551e2c6c44d187176a42cdda50c2] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=47.0 K 2024-11-08T02:24:41,487 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/ba0ef423fd664345af427ac8e4a45adb as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/ba0ef423fd664345af427ac8e4a45adb 2024-11-08T02:24:41,488 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb9d55ca77ec4ee59d55ab0d4971b328, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731032676030 2024-11-08T02:24:41,488 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c83fece32e364ed79d9b30661c63fcc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731032677102 2024-11-08T02:24:41,488 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ce696d3cb664845a2dd722ade88738c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731032678347 2024-11-08T02:24:41,488 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1bd551e2c6c44d187176a42cdda50c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731032679440 2024-11-08T02:24:41,494 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9024be227c18091a068b12c5acebba9f/B of 9024be227c18091a068b12c5acebba9f into ba0ef423fd664345af427ac8e4a45adb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:41,494 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:41,494 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/B, priority=12, startTime=1731032681453; duration=0sec 2024-11-08T02:24:41,494 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:41,494 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:B 2024-11-08T02:24:41,498 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#C#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:41,498 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/2da01661256046f7aad5179a0ea80560 is 50, key is test_row_0/C:col10/1731032680211/Put/seqid=0 2024-11-08T02:24:41,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742289_1465 (size=12241) 2024-11-08T02:24:41,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T02:24:41,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:41,588 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-08T02:24:41,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/75eb4f78c42a416c897265205bfaa77c is 50, key is test_row_0/A:col10/1731032680236/Put/seqid=0 2024-11-08T02:24:41,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742290_1466 (size=9657) 2024-11-08T02:24:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:41,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:41,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032741723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T02:24:41,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032741828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:41,915 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/2da01661256046f7aad5179a0ea80560 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2da01661256046f7aad5179a0ea80560 2024-11-08T02:24:41,919 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9024be227c18091a068b12c5acebba9f/C of 9024be227c18091a068b12c5acebba9f into 2da01661256046f7aad5179a0ea80560(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:41,919 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:41,919 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/C, priority=12, startTime=1731032681453; duration=0sec 2024-11-08T02:24:41,919 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:41,919 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:C 2024-11-08T02:24:41,995 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/75eb4f78c42a416c897265205bfaa77c 2024-11-08T02:24:42,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/4cf6cb704aaf4543bce4b63b66554997 is 50, key is test_row_0/B:col10/1731032680236/Put/seqid=0 2024-11-08T02:24:42,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742291_1467 (size=9657) 2024-11-08T02:24:42,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032742038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:42,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T02:24:42,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032742358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:42,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:42,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032742388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:42,392 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:42,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032742398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:42,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032742398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:42,403 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:42,403 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:42,445 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/4cf6cb704aaf4543bce4b63b66554997 2024-11-08T02:24:42,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/1558335d8a5949ea9049a8402c8e8f0a is 50, key is test_row_0/C:col10/1731032680236/Put/seqid=0 2024-11-08T02:24:42,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742292_1468 (size=9657) 2024-11-08T02:24:42,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:42,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032742547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:42,551 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:42,855 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/1558335d8a5949ea9049a8402c8e8f0a 2024-11-08T02:24:42,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/75eb4f78c42a416c897265205bfaa77c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/75eb4f78c42a416c897265205bfaa77c 2024-11-08T02:24:42,860 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/75eb4f78c42a416c897265205bfaa77c, entries=100, sequenceid=126, filesize=9.4 K 2024-11-08T02:24:42,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/4cf6cb704aaf4543bce4b63b66554997 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/4cf6cb704aaf4543bce4b63b66554997 2024-11-08T02:24:42,864 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/4cf6cb704aaf4543bce4b63b66554997, entries=100, sequenceid=126, filesize=9.4 K 2024-11-08T02:24:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/1558335d8a5949ea9049a8402c8e8f0a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/1558335d8a5949ea9049a8402c8e8f0a 2024-11-08T02:24:42,867 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/1558335d8a5949ea9049a8402c8e8f0a, entries=100, sequenceid=126, filesize=9.4 K 2024-11-08T02:24:42,868 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 9024be227c18091a068b12c5acebba9f in 1280ms, sequenceid=126, compaction requested=false 2024-11-08T02:24:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-08T02:24:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-08T02:24:42,871 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-08T02:24:42,871 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7390 sec 2024-11-08T02:24:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:42,872 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:42,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T02:24:42,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.7430 sec 2024-11-08T02:24:42,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/5d11697cf0794a18a8b3e5e339081c84 is 50, key is test_row_0/A:col10/1731032681704/Put/seqid=0 2024-11-08T02:24:42,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742293_1469 (size=14541) 2024-11-08T02:24:42,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032742896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:43,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032743008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:43,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:43,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:43,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032743218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:43,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T02:24:43,278 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:43,279 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-08T02:24:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T02:24:43,280 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:43,281 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:43,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:43,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/5d11697cf0794a18a8b3e5e339081c84 2024-11-08T02:24:43,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/bb839d6423c1429c976134c12e37f9c0 is 50, key is test_row_0/B:col10/1731032681704/Put/seqid=0 2024-11-08T02:24:43,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742294_1470 (size=12151) 2024-11-08T02:24:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T02:24:43,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T02:24:43,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:43,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:43,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:43,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:43,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032743528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:43,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T02:24:43,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:43,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:43,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:43,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T02:24:43,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/bb839d6423c1429c976134c12e37f9c0 2024-11-08T02:24:43,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c26f1c396e62449cb86b404a28bb96bd is 50, key is test_row_0/C:col10/1731032681704/Put/seqid=0 2024-11-08T02:24:43,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742295_1471 (size=12151) 2024-11-08T02:24:43,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T02:24:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:43,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T02:24:43,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:43,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:43,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:43,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T02:24:44,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032744038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T02:24:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:44,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:44,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c26f1c396e62449cb86b404a28bb96bd 2024-11-08T02:24:44,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/5d11697cf0794a18a8b3e5e339081c84 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/5d11697cf0794a18a8b3e5e339081c84 2024-11-08T02:24:44,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/5d11697cf0794a18a8b3e5e339081c84, entries=200, sequenceid=155, filesize=14.2 K 2024-11-08T02:24:44,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/bb839d6423c1429c976134c12e37f9c0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/bb839d6423c1429c976134c12e37f9c0 2024-11-08T02:24:44,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/bb839d6423c1429c976134c12e37f9c0, entries=150, sequenceid=155, filesize=11.9 K 2024-11-08T02:24:44,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c26f1c396e62449cb86b404a28bb96bd as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c26f1c396e62449cb86b404a28bb96bd 2024-11-08T02:24:44,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c26f1c396e62449cb86b404a28bb96bd, entries=150, sequenceid=155, filesize=11.9 K 2024-11-08T02:24:44,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 9024be227c18091a068b12c5acebba9f in 1245ms, sequenceid=155, compaction requested=true 2024-11-08T02:24:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:44,117 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:44,117 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:44,118 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:44,118 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:44,118 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/B is initiating minor compaction (all files) 2024-11-08T02:24:44,118 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/A is initiating minor compaction (all files) 2024-11-08T02:24:44,118 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/B in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:44,118 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/A in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:44,118 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/ba0ef423fd664345af427ac8e4a45adb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/4cf6cb704aaf4543bce4b63b66554997, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/bb839d6423c1429c976134c12e37f9c0] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=33.3 K 2024-11-08T02:24:44,118 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/62b92e61a18d4551b94af4741694877f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/75eb4f78c42a416c897265205bfaa77c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/5d11697cf0794a18a8b3e5e339081c84] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=35.6 K 2024-11-08T02:24:44,118 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting ba0ef423fd664345af427ac8e4a45adb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731032679440 2024-11-08T02:24:44,118 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62b92e61a18d4551b94af4741694877f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731032679440 2024-11-08T02:24:44,118 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cf6cb704aaf4543bce4b63b66554997, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731032680226 2024-11-08T02:24:44,118 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75eb4f78c42a416c897265205bfaa77c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731032680226 2024-11-08T02:24:44,119 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting bb839d6423c1429c976134c12e37f9c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731032681699 2024-11-08T02:24:44,119 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d11697cf0794a18a8b3e5e339081c84, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731032681699 2024-11-08T02:24:44,125 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:44,125 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/a5d1bf5b487543f6a28d902f4784a26f is 50, key is test_row_0/B:col10/1731032681704/Put/seqid=0 2024-11-08T02:24:44,125 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#A#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:44,126 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/938b011f004c445a9dffa587ad4f5cdc is 50, key is test_row_0/A:col10/1731032681704/Put/seqid=0 2024-11-08T02:24:44,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742296_1472 (size=12493) 2024-11-08T02:24:44,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742297_1473 (size=12493) 2024-11-08T02:24:44,153 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/a5d1bf5b487543f6a28d902f4784a26f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/a5d1bf5b487543f6a28d902f4784a26f 2024-11-08T02:24:44,157 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/B of 9024be227c18091a068b12c5acebba9f into a5d1bf5b487543f6a28d902f4784a26f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:44,157 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:44,157 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/B, priority=13, startTime=1731032684117; duration=0sec 2024-11-08T02:24:44,157 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:44,157 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:B 2024-11-08T02:24:44,157 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:44,158 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:44,158 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/C is initiating minor compaction (all files) 2024-11-08T02:24:44,158 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/C in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:44,158 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2da01661256046f7aad5179a0ea80560, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/1558335d8a5949ea9049a8402c8e8f0a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c26f1c396e62449cb86b404a28bb96bd] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=33.3 K 2024-11-08T02:24:44,158 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 2da01661256046f7aad5179a0ea80560, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731032679440 2024-11-08T02:24:44,158 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 1558335d8a5949ea9049a8402c8e8f0a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731032680226 2024-11-08T02:24:44,159 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c26f1c396e62449cb86b404a28bb96bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731032681699 2024-11-08T02:24:44,164 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:44,164 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/6bcdbc5b0f824558ad65b334a68b7326 is 50, key is test_row_0/C:col10/1731032681704/Put/seqid=0 2024-11-08T02:24:44,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742298_1474 (size=12493) 2024-11-08T02:24:44,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T02:24:44,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:44,196 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T02:24:44,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/0b04b00f39cf430d9f61da7eef5b1088 is 50, key is test_row_0/A:col10/1731032682895/Put/seqid=0 2024-11-08T02:24:44,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742299_1475 (size=12151) 2024-11-08T02:24:44,205 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/0b04b00f39cf430d9f61da7eef5b1088 2024-11-08T02:24:44,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/91c09257964a474f942f83d7646b0057 is 50, key is test_row_0/B:col10/1731032682895/Put/seqid=0 2024-11-08T02:24:44,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742300_1476 (size=12151) 2024-11-08T02:24:44,217 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/91c09257964a474f942f83d7646b0057 2024-11-08T02:24:44,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/8b46f2a6941540f68cbe0ea099186ae7 is 50, key is test_row_0/C:col10/1731032682895/Put/seqid=0 2024-11-08T02:24:44,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742301_1477 (size=12151) 2024-11-08T02:24:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:44,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T02:24:44,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032744458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032744460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032744461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,543 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/938b011f004c445a9dffa587ad4f5cdc as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/938b011f004c445a9dffa587ad4f5cdc 2024-11-08T02:24:44,546 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/A of 9024be227c18091a068b12c5acebba9f into 938b011f004c445a9dffa587ad4f5cdc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:44,546 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:44,546 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/A, priority=13, startTime=1731032684117; duration=0sec 2024-11-08T02:24:44,546 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:44,546 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:A 2024-11-08T02:24:44,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032744567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032744567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,575 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/6bcdbc5b0f824558ad65b334a68b7326 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/6bcdbc5b0f824558ad65b334a68b7326 2024-11-08T02:24:44,578 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/C of 9024be227c18091a068b12c5acebba9f into 6bcdbc5b0f824558ad65b334a68b7326(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:44,578 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:44,578 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/C, priority=13, startTime=1731032684117; duration=0sec 2024-11-08T02:24:44,579 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:44,579 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:C 2024-11-08T02:24:44,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032744578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,629 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/8b46f2a6941540f68cbe0ea099186ae7 2024-11-08T02:24:44,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/0b04b00f39cf430d9f61da7eef5b1088 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/0b04b00f39cf430d9f61da7eef5b1088 2024-11-08T02:24:44,636 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/0b04b00f39cf430d9f61da7eef5b1088, entries=150, sequenceid=164, filesize=11.9 K 2024-11-08T02:24:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/91c09257964a474f942f83d7646b0057 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/91c09257964a474f942f83d7646b0057 2024-11-08T02:24:44,641 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/91c09257964a474f942f83d7646b0057, entries=150, sequenceid=164, filesize=11.9 K 2024-11-08T02:24:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/8b46f2a6941540f68cbe0ea099186ae7 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8b46f2a6941540f68cbe0ea099186ae7 2024-11-08T02:24:44,645 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8b46f2a6941540f68cbe0ea099186ae7, entries=150, sequenceid=164, filesize=11.9 K 2024-11-08T02:24:44,646 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 9024be227c18091a068b12c5acebba9f in 449ms, sequenceid=164, compaction requested=false 2024-11-08T02:24:44,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:44,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:44,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-08T02:24:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-08T02:24:44,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-08T02:24:44,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3660 sec 2024-11-08T02:24:44,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3730 sec 2024-11-08T02:24:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:44,782 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:44,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-08T02:24:44,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/62fb900099b54417bdc2addfb2a24c88 is 50, key is test_row_0/A:col10/1731032684433/Put/seqid=0 2024-11-08T02:24:44,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742302_1478 (size=14541) 2024-11-08T02:24:44,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/62fb900099b54417bdc2addfb2a24c88 2024-11-08T02:24:44,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032744789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032744790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032744792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/dc86efb10d094b65aebbe52101fb4962 is 50, key is test_row_0/B:col10/1731032684433/Put/seqid=0 2024-11-08T02:24:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742303_1479 (size=12151) 2024-11-08T02:24:44,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032744898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032744908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:44,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032744908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:44,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:44,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:44,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:45,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032745057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:45,061 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:45,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032745108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:45,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:45,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032745118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:45,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032745118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:45,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/dc86efb10d094b65aebbe52101fb4962 2024-11-08T02:24:45,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/55d68927babc49c292b057a69841d33a is 50, key is test_row_0/C:col10/1731032684433/Put/seqid=0 2024-11-08T02:24:45,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742304_1480 (size=12151) 2024-11-08T02:24:45,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032745417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:45,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T02:24:45,428 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:45,428 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-08T02:24:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T02:24:45,430 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:45,430 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:45,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:45,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032745428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:45,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032745428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T02:24:45,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T02:24:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:45,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:45,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/55d68927babc49c292b057a69841d33a 2024-11-08T02:24:45,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/62fb900099b54417bdc2addfb2a24c88 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/62fb900099b54417bdc2addfb2a24c88 2024-11-08T02:24:45,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/62fb900099b54417bdc2addfb2a24c88, entries=200, sequenceid=195, filesize=14.2 K 2024-11-08T02:24:45,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/dc86efb10d094b65aebbe52101fb4962 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/dc86efb10d094b65aebbe52101fb4962 2024-11-08T02:24:45,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/dc86efb10d094b65aebbe52101fb4962, entries=150, sequenceid=195, filesize=11.9 K 2024-11-08T02:24:45,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/55d68927babc49c292b057a69841d33a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/55d68927babc49c292b057a69841d33a 2024-11-08T02:24:45,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/55d68927babc49c292b057a69841d33a, entries=150, sequenceid=195, filesize=11.9 K 2024-11-08T02:24:45,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 9024be227c18091a068b12c5acebba9f in 849ms, sequenceid=195, compaction requested=true 2024-11-08T02:24:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:45,631 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:45,631 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:45,632 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:45,632 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:45,632 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/B is initiating minor compaction (all files) 2024-11-08T02:24:45,632 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/A is initiating minor compaction (all files) 2024-11-08T02:24:45,632 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/A in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:45,632 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/B in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:45,632 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/a5d1bf5b487543f6a28d902f4784a26f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/91c09257964a474f942f83d7646b0057, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/dc86efb10d094b65aebbe52101fb4962] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=35.9 K 2024-11-08T02:24:45,632 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/938b011f004c445a9dffa587ad4f5cdc, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/0b04b00f39cf430d9f61da7eef5b1088, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/62fb900099b54417bdc2addfb2a24c88] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=38.3 K 2024-11-08T02:24:45,633 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a5d1bf5b487543f6a28d902f4784a26f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731032681699 2024-11-08T02:24:45,633 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 938b011f004c445a9dffa587ad4f5cdc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731032681699 2024-11-08T02:24:45,633 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 91c09257964a474f942f83d7646b0057, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731032682880 2024-11-08T02:24:45,633 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b04b00f39cf430d9f61da7eef5b1088, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731032682880 2024-11-08T02:24:45,633 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting dc86efb10d094b65aebbe52101fb4962, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032684433 2024-11-08T02:24:45,633 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62fb900099b54417bdc2addfb2a24c88, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032684433 2024-11-08T02:24:45,645 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:45,645 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:45,646 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/37c315ff9e654602aa6434a42e7a07ae is 50, key is test_row_0/B:col10/1731032684433/Put/seqid=0 2024-11-08T02:24:45,646 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/a6ea7c7727894565a1e6b3009a143c8e is 50, key is test_row_0/A:col10/1731032684433/Put/seqid=0 2024-11-08T02:24:45,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742306_1482 (size=12595) 2024-11-08T02:24:45,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742305_1481 (size=12595) 2024-11-08T02:24:45,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T02:24:45,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:45,735 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T02:24:45,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/d3e8d58e2ca54e53acad8e8780f3bce5 is 50, key is test_row_0/A:col10/1731032684791/Put/seqid=0 2024-11-08T02:24:45,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742307_1483 (size=9757) 2024-11-08T02:24:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T02:24:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:45,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:45,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032745990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:45,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032745991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:45,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032745992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,056 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/a6ea7c7727894565a1e6b3009a143c8e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/a6ea7c7727894565a1e6b3009a143c8e 2024-11-08T02:24:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T02:24:46,060 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/A of 9024be227c18091a068b12c5acebba9f into a6ea7c7727894565a1e6b3009a143c8e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:46,060 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:46,060 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/A, priority=13, startTime=1731032685631; duration=0sec 2024-11-08T02:24:46,060 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:46,060 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:A 2024-11-08T02:24:46,060 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:46,061 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:46,061 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/C is initiating minor compaction (all files) 2024-11-08T02:24:46,061 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/C in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:46,061 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/6bcdbc5b0f824558ad65b334a68b7326, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8b46f2a6941540f68cbe0ea099186ae7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/55d68927babc49c292b057a69841d33a] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=35.9 K 2024-11-08T02:24:46,062 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bcdbc5b0f824558ad65b334a68b7326, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731032681699 2024-11-08T02:24:46,062 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b46f2a6941540f68cbe0ea099186ae7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731032682880 2024-11-08T02:24:46,062 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55d68927babc49c292b057a69841d33a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032684433 2024-11-08T02:24:46,068 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/37c315ff9e654602aa6434a42e7a07ae as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/37c315ff9e654602aa6434a42e7a07ae 2024-11-08T02:24:46,068 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:46,068 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/3d8b8a5c8c7c48088ede857aadfe4a19 is 50, key is test_row_0/C:col10/1731032684433/Put/seqid=0 2024-11-08T02:24:46,071 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/B of 9024be227c18091a068b12c5acebba9f into 37c315ff9e654602aa6434a42e7a07ae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:46,071 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:46,071 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/B, priority=13, startTime=1731032685631; duration=0sec 2024-11-08T02:24:46,071 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:46,071 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:B 2024-11-08T02:24:46,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742308_1484 (size=12595) 2024-11-08T02:24:46,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032746108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:46,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032746108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032746108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,141 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/d3e8d58e2ca54e53acad8e8780f3bce5 2024-11-08T02:24:46,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/edaa0bfb091c4367a84ac61b69f33d05 is 50, key is test_row_0/B:col10/1731032684791/Put/seqid=0 2024-11-08T02:24:46,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742309_1485 (size=9757) 2024-11-08T02:24:46,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032746317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032746317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032746318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,477 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/3d8b8a5c8c7c48088ede857aadfe4a19 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/3d8b8a5c8c7c48088ede857aadfe4a19 2024-11-08T02:24:46,480 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/C of 9024be227c18091a068b12c5acebba9f into 3d8b8a5c8c7c48088ede857aadfe4a19(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:46,481 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:46,481 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/C, priority=13, startTime=1731032685631; duration=0sec 2024-11-08T02:24:46,481 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:46,481 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:C 2024-11-08T02:24:46,552 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/edaa0bfb091c4367a84ac61b69f33d05 2024-11-08T02:24:46,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c6f220924b124f72bb06fbd8f7017754 is 50, key is test_row_0/C:col10/1731032684791/Put/seqid=0 2024-11-08T02:24:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T02:24:46,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742310_1486 (size=9757) 2024-11-08T02:24:46,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47120 deadline: 1731032746588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:46,595 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032746628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:46,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032746628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032746628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:46,970 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c6f220924b124f72bb06fbd8f7017754 2024-11-08T02:24:46,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/d3e8d58e2ca54e53acad8e8780f3bce5 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/d3e8d58e2ca54e53acad8e8780f3bce5 2024-11-08T02:24:46,976 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/d3e8d58e2ca54e53acad8e8780f3bce5, entries=100, sequenceid=202, filesize=9.5 K 2024-11-08T02:24:46,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/edaa0bfb091c4367a84ac61b69f33d05 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/edaa0bfb091c4367a84ac61b69f33d05 2024-11-08T02:24:46,980 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/edaa0bfb091c4367a84ac61b69f33d05, entries=100, sequenceid=202, filesize=9.5 K 2024-11-08T02:24:46,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c6f220924b124f72bb06fbd8f7017754 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c6f220924b124f72bb06fbd8f7017754 2024-11-08T02:24:46,985 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c6f220924b124f72bb06fbd8f7017754, entries=100, sequenceid=202, filesize=9.5 K 2024-11-08T02:24:46,985 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 9024be227c18091a068b12c5acebba9f in 1251ms, sequenceid=202, compaction requested=false 2024-11-08T02:24:46,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:46,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:46,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-08T02:24:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-08T02:24:46,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-08T02:24:46,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5570 sec 2024-11-08T02:24:46,990 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5610 sec 2024-11-08T02:24:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:47,072 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:47,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-08T02:24:47,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/4500ec57c1bb48c1b5821c31f77a54d5 is 50, key is test_row_0/A:col10/1731032687071/Put/seqid=0 2024-11-08T02:24:47,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742311_1487 (size=14541) 2024-11-08T02:24:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032747089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:47,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032747148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:47,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032747148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:47,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032747148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:47,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032747197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:47,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032747408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:47,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/4500ec57c1bb48c1b5821c31f77a54d5 2024-11-08T02:24:47,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/2106676296e44647b5dbaaff4346343a is 50, key is test_row_0/B:col10/1731032687071/Put/seqid=0 2024-11-08T02:24:47,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742312_1488 (size=12151) 2024-11-08T02:24:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T02:24:47,579 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:47,579 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-08T02:24:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T02:24:47,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:47,582 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:47,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T02:24:47,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032747717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:47,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T02:24:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:47,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:47,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T02:24:47,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:47,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:47,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:47,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:47,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:47,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/2106676296e44647b5dbaaff4346343a 2024-11-08T02:24:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T02:24:47,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/601dc582a67346cc89012c9b3d998c11 is 50, key is test_row_0/C:col10/1731032687071/Put/seqid=0 2024-11-08T02:24:47,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742313_1489 (size=12151) 2024-11-08T02:24:48,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T02:24:48,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:48,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:48,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:48,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:48,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:48,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032748158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:48,159 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:48,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032748167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:48,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032748168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:48,173 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:48,173 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:48,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:48,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T02:24:48,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:48,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:48,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:48,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:48,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T02:24:48,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032748231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:48,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:48,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:48,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:48,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/601dc582a67346cc89012c9b3d998c11 2024-11-08T02:24:48,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/4500ec57c1bb48c1b5821c31f77a54d5 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/4500ec57c1bb48c1b5821c31f77a54d5 2024-11-08T02:24:48,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/4500ec57c1bb48c1b5821c31f77a54d5, entries=200, sequenceid=235, filesize=14.2 K 2024-11-08T02:24:48,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/2106676296e44647b5dbaaff4346343a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/2106676296e44647b5dbaaff4346343a 2024-11-08T02:24:48,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/2106676296e44647b5dbaaff4346343a, entries=150, sequenceid=235, filesize=11.9 K 2024-11-08T02:24:48,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/601dc582a67346cc89012c9b3d998c11 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/601dc582a67346cc89012c9b3d998c11 2024-11-08T02:24:48,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T02:24:48,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:48,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:48,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:48,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:48,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:48,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/601dc582a67346cc89012c9b3d998c11, entries=150, sequenceid=235, filesize=11.9 K 2024-11-08T02:24:48,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 9024be227c18091a068b12c5acebba9f in 1276ms, sequenceid=235, compaction requested=true 2024-11-08T02:24:48,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:48,348 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:48,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T02:24:48,352 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36893 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:48,352 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/A is initiating minor compaction (all files) 2024-11-08T02:24:48,352 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/A in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:48,353 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/a6ea7c7727894565a1e6b3009a143c8e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/d3e8d58e2ca54e53acad8e8780f3bce5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/4500ec57c1bb48c1b5821c31f77a54d5] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=36.0 K 2024-11-08T02:24:48,354 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6ea7c7727894565a1e6b3009a143c8e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032684433 2024-11-08T02:24:48,354 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3e8d58e2ca54e53acad8e8780f3bce5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731032684788 2024-11-08T02:24:48,354 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4500ec57c1bb48c1b5821c31f77a54d5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032685986 2024-11-08T02:24:48,358 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:48,361 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:48,361 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/B is initiating minor compaction (all files) 2024-11-08T02:24:48,361 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/B in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:48,361 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/37c315ff9e654602aa6434a42e7a07ae, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/edaa0bfb091c4367a84ac61b69f33d05, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/2106676296e44647b5dbaaff4346343a] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=33.7 K 2024-11-08T02:24:48,361 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 37c315ff9e654602aa6434a42e7a07ae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032684433 2024-11-08T02:24:48,362 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting edaa0bfb091c4367a84ac61b69f33d05, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731032684788 2024-11-08T02:24:48,363 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 2106676296e44647b5dbaaff4346343a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032685986 2024-11-08T02:24:48,371 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:48,372 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/c610a65e07f4446c80bb78647691a769 is 50, key is test_row_0/A:col10/1731032687071/Put/seqid=0 2024-11-08T02:24:48,373 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:48,373 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/b85dd5cb27e945fbbc416f3915025ad8 is 50, key is test_row_0/B:col10/1731032687071/Put/seqid=0 2024-11-08T02:24:48,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742314_1490 (size=12697) 2024-11-08T02:24:48,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742315_1491 (size=12697) 2024-11-08T02:24:48,391 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/b85dd5cb27e945fbbc416f3915025ad8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b85dd5cb27e945fbbc416f3915025ad8 2024-11-08T02:24:48,396 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/B of 9024be227c18091a068b12c5acebba9f into b85dd5cb27e945fbbc416f3915025ad8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:48,397 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:48,397 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/B, priority=13, startTime=1731032688349; duration=0sec 2024-11-08T02:24:48,397 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:48,397 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:B 2024-11-08T02:24:48,397 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:48,398 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:48,398 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/C is initiating minor compaction (all files) 2024-11-08T02:24:48,398 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/C in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:48,398 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/3d8b8a5c8c7c48088ede857aadfe4a19, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c6f220924b124f72bb06fbd8f7017754, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/601dc582a67346cc89012c9b3d998c11] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=33.7 K 2024-11-08T02:24:48,398 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d8b8a5c8c7c48088ede857aadfe4a19, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032684433 2024-11-08T02:24:48,398 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c6f220924b124f72bb06fbd8f7017754, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731032684788 2024-11-08T02:24:48,399 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 601dc582a67346cc89012c9b3d998c11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032685986 2024-11-08T02:24:48,406 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:48,406 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c84a3d5509fc4f14a07364089d61fb3d is 50, key is test_row_0/C:col10/1731032687071/Put/seqid=0 2024-11-08T02:24:48,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742316_1492 (size=12697) 2024-11-08T02:24:48,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T02:24:48,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:48,499 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T02:24:48,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/cdb78eac0dae4f199be3bb1fe0e91f34 is 50, key is test_row_0/A:col10/1731032687087/Put/seqid=0 2024-11-08T02:24:48,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742317_1493 (size=12151) 2024-11-08T02:24:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T02:24:48,792 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/c610a65e07f4446c80bb78647691a769 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/c610a65e07f4446c80bb78647691a769 2024-11-08T02:24:48,796 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/A of 9024be227c18091a068b12c5acebba9f into c610a65e07f4446c80bb78647691a769(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:48,796 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:48,797 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/A, priority=13, startTime=1731032688348; duration=0sec 2024-11-08T02:24:48,797 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:48,797 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:A 2024-11-08T02:24:48,816 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c84a3d5509fc4f14a07364089d61fb3d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c84a3d5509fc4f14a07364089d61fb3d 2024-11-08T02:24:48,821 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/C of 9024be227c18091a068b12c5acebba9f into c84a3d5509fc4f14a07364089d61fb3d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:48,821 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:48,821 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/C, priority=13, startTime=1731032688349; duration=0sec 2024-11-08T02:24:48,821 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:48,821 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:C 2024-11-08T02:24:48,931 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/cdb78eac0dae4f199be3bb1fe0e91f34 2024-11-08T02:24:48,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/b7e5ddd61b494c3aa9028286722df985 is 50, key is test_row_0/B:col10/1731032687087/Put/seqid=0 2024-11-08T02:24:48,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742318_1494 (size=12151) 2024-11-08T02:24:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:49,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:49,350 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/b7e5ddd61b494c3aa9028286722df985 2024-11-08T02:24:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/5edb88e70b6e4f79ad3ffa6726d001e0 is 50, key is test_row_0/C:col10/1731032687087/Put/seqid=0 2024-11-08T02:24:49,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742319_1495 (size=12151) 2024-11-08T02:24:49,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:49,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032749380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:49,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:49,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:49,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032749488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:49,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:49,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:49,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:49,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032749697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:49,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:49,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:49,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T02:24:49,758 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/5edb88e70b6e4f79ad3ffa6726d001e0 2024-11-08T02:24:49,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/cdb78eac0dae4f199be3bb1fe0e91f34 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cdb78eac0dae4f199be3bb1fe0e91f34 2024-11-08T02:24:49,764 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cdb78eac0dae4f199be3bb1fe0e91f34, entries=150, sequenceid=242, filesize=11.9 K 2024-11-08T02:24:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/b7e5ddd61b494c3aa9028286722df985 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b7e5ddd61b494c3aa9028286722df985 2024-11-08T02:24:49,767 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b7e5ddd61b494c3aa9028286722df985, entries=150, sequenceid=242, filesize=11.9 K 2024-11-08T02:24:49,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/5edb88e70b6e4f79ad3ffa6726d001e0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/5edb88e70b6e4f79ad3ffa6726d001e0 2024-11-08T02:24:49,771 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/5edb88e70b6e4f79ad3ffa6726d001e0, entries=150, sequenceid=242, filesize=11.9 K 2024-11-08T02:24:49,771 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 9024be227c18091a068b12c5acebba9f in 1272ms, sequenceid=242, compaction requested=false 2024-11-08T02:24:49,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:49,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:49,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-08T02:24:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-08T02:24:49,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-08T02:24:49,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1910 sec 2024-11-08T02:24:49,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.1950 sec 2024-11-08T02:24:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:50,012 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:50,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-08T02:24:50,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/3926df139b39470dbdd6125d72833e10 is 50, key is test_row_0/A:col10/1731032690011/Put/seqid=0 2024-11-08T02:24:50,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742320_1496 (size=14741) 2024-11-08T02:24:50,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032750021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:50,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:50,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032750138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:50,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032750177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:50,180 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:50,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032750198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:50,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032750198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:50,200 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:50,200 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:50,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:50,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032750348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:50,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/3926df139b39470dbdd6125d72833e10 2024-11-08T02:24:50,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/cc2482624d204ec0a305d8bb22bb26d7 is 50, key is test_row_0/B:col10/1731032690011/Put/seqid=0 2024-11-08T02:24:50,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742321_1497 (size=12301) 2024-11-08T02:24:50,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/cc2482624d204ec0a305d8bb22bb26d7 2024-11-08T02:24:50,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/2c00f682a203447d868f1ff02eab0c76 is 50, key is test_row_0/C:col10/1731032690011/Put/seqid=0 2024-11-08T02:24:50,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742322_1498 (size=12301) 2024-11-08T02:24:50,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/2c00f682a203447d868f1ff02eab0c76 2024-11-08T02:24:50,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/3926df139b39470dbdd6125d72833e10 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/3926df139b39470dbdd6125d72833e10 2024-11-08T02:24:50,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/3926df139b39470dbdd6125d72833e10, entries=200, sequenceid=275, filesize=14.4 K 2024-11-08T02:24:50,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/cc2482624d204ec0a305d8bb22bb26d7 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/cc2482624d204ec0a305d8bb22bb26d7 2024-11-08T02:24:50,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/cc2482624d204ec0a305d8bb22bb26d7, entries=150, sequenceid=275, filesize=12.0 K 2024-11-08T02:24:50,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/2c00f682a203447d868f1ff02eab0c76 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2c00f682a203447d868f1ff02eab0c76 2024-11-08T02:24:50,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2c00f682a203447d868f1ff02eab0c76, entries=150, sequenceid=275, filesize=12.0 K 2024-11-08T02:24:50,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 9024be227c18091a068b12c5acebba9f in 447ms, sequenceid=275, compaction requested=true 2024-11-08T02:24:50,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:50,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:50,459 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:50,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:50,459 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:50,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:50,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:50,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:50,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:50,460 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:50,460 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:50,460 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/B is initiating minor compaction (all files) 2024-11-08T02:24:50,460 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/A is initiating minor compaction (all files) 2024-11-08T02:24:50,460 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/A in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:50,460 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/B in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:50,461 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b85dd5cb27e945fbbc416f3915025ad8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b7e5ddd61b494c3aa9028286722df985, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/cc2482624d204ec0a305d8bb22bb26d7] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=36.3 K 2024-11-08T02:24:50,461 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/c610a65e07f4446c80bb78647691a769, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cdb78eac0dae4f199be3bb1fe0e91f34, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/3926df139b39470dbdd6125d72833e10] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=38.7 K 2024-11-08T02:24:50,461 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting c610a65e07f4446c80bb78647691a769, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032685986 2024-11-08T02:24:50,461 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting b85dd5cb27e945fbbc416f3915025ad8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032685986 2024-11-08T02:24:50,461 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdb78eac0dae4f199be3bb1fe0e91f34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731032687078 2024-11-08T02:24:50,462 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e5ddd61b494c3aa9028286722df985, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731032687078 2024-11-08T02:24:50,462 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3926df139b39470dbdd6125d72833e10, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731032689364 2024-11-08T02:24:50,462 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting cc2482624d204ec0a305d8bb22bb26d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731032689364 2024-11-08T02:24:50,470 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:50,471 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#A#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:50,471 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/05d59e80bba04e31b10444b4a511b08f is 50, key is test_row_0/B:col10/1731032690011/Put/seqid=0 2024-11-08T02:24:50,471 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/27dea432956b40e39612368df8ea23a6 is 50, key is test_row_0/A:col10/1731032690011/Put/seqid=0 2024-11-08T02:24:50,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742323_1499 (size=12949) 2024-11-08T02:24:50,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742324_1500 (size=12949) 2024-11-08T02:24:50,486 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/05d59e80bba04e31b10444b4a511b08f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/05d59e80bba04e31b10444b4a511b08f 2024-11-08T02:24:50,490 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/27dea432956b40e39612368df8ea23a6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/27dea432956b40e39612368df8ea23a6 2024-11-08T02:24:50,491 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/B of 9024be227c18091a068b12c5acebba9f into 05d59e80bba04e31b10444b4a511b08f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:50,491 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:50,491 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/B, priority=13, startTime=1731032690459; duration=0sec 2024-11-08T02:24:50,492 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:50,492 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:B 2024-11-08T02:24:50,492 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:24:50,493 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:24:50,493 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/C is initiating minor compaction (all files) 2024-11-08T02:24:50,493 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/C in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:50,493 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c84a3d5509fc4f14a07364089d61fb3d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/5edb88e70b6e4f79ad3ffa6726d001e0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2c00f682a203447d868f1ff02eab0c76] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=36.3 K 2024-11-08T02:24:50,493 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c84a3d5509fc4f14a07364089d61fb3d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032685986 2024-11-08T02:24:50,493 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 5edb88e70b6e4f79ad3ffa6726d001e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731032687078 2024-11-08T02:24:50,493 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c00f682a203447d868f1ff02eab0c76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731032689364 2024-11-08T02:24:50,495 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/A of 9024be227c18091a068b12c5acebba9f into 27dea432956b40e39612368df8ea23a6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:50,496 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:50,496 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/A, priority=13, startTime=1731032690459; duration=0sec 2024-11-08T02:24:50,496 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:50,496 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:A 2024-11-08T02:24:50,502 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#C#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:50,502 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/f2f785a411c74b09958df5b246dfa066 is 50, key is test_row_0/C:col10/1731032690011/Put/seqid=0 2024-11-08T02:24:50,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742325_1501 (size=12949) 2024-11-08T02:24:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:50,673 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:50,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:24:50,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/e77297b378d4408186be828ad44c7e1d is 50, key is test_row_0/A:col10/1731032690672/Put/seqid=0 2024-11-08T02:24:50,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742326_1502 (size=14741) 2024-11-08T02:24:50,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032750773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:50,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032750878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:50,919 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/f2f785a411c74b09958df5b246dfa066 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/f2f785a411c74b09958df5b246dfa066 2024-11-08T02:24:50,927 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9024be227c18091a068b12c5acebba9f/C of 9024be227c18091a068b12c5acebba9f into f2f785a411c74b09958df5b246dfa066(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:50,927 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:50,927 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/C, priority=13, startTime=1731032690459; duration=0sec 2024-11-08T02:24:50,927 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:50,927 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:C 2024-11-08T02:24:51,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/e77297b378d4408186be828ad44c7e1d 2024-11-08T02:24:51,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/723244f8634e46cc96f00a0bf5be0922 is 50, key is test_row_0/B:col10/1731032690672/Put/seqid=0 2024-11-08T02:24:51,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742327_1503 (size=12301) 2024-11-08T02:24:51,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032751088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:51,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:51,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032751398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:51,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/723244f8634e46cc96f00a0bf5be0922 2024-11-08T02:24:51,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/f00e92cd5c724f5f9258aec86b104b03 is 50, key is test_row_0/C:col10/1731032690672/Put/seqid=0 2024-11-08T02:24:51,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742328_1504 (size=12301) 2024-11-08T02:24:51,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/f00e92cd5c724f5f9258aec86b104b03 2024-11-08T02:24:51,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/e77297b378d4408186be828ad44c7e1d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/e77297b378d4408186be828ad44c7e1d 2024-11-08T02:24:51,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/e77297b378d4408186be828ad44c7e1d, entries=200, sequenceid=288, filesize=14.4 K 2024-11-08T02:24:51,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/723244f8634e46cc96f00a0bf5be0922 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/723244f8634e46cc96f00a0bf5be0922 2024-11-08T02:24:51,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/723244f8634e46cc96f00a0bf5be0922, entries=150, sequenceid=288, filesize=12.0 K 2024-11-08T02:24:51,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/f00e92cd5c724f5f9258aec86b104b03 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/f00e92cd5c724f5f9258aec86b104b03 2024-11-08T02:24:51,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/f00e92cd5c724f5f9258aec86b104b03, entries=150, sequenceid=288, filesize=12.0 K 2024-11-08T02:24:51,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9024be227c18091a068b12c5acebba9f in 849ms, sequenceid=288, compaction requested=false 2024-11-08T02:24:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T02:24:51,738 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:51,739 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-08T02:24:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T02:24:51,740 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:51,741 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:51,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T02:24:51,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-08T02:24:51,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:51,893 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:24:51,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/f83107bcb708450faee2433335653de4 is 50, key is test_row_0/A:col10/1731032690755/Put/seqid=0 2024-11-08T02:24:51,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742329_1505 (size=12301) 2024-11-08T02:24:51,906 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/f83107bcb708450faee2433335653de4 2024-11-08T02:24:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:51,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:51,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/b76faad89e374ded8b9b339f36e518f3 is 50, key is test_row_0/B:col10/1731032690755/Put/seqid=0 2024-11-08T02:24:51,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742330_1506 (size=12301) 2024-11-08T02:24:51,917 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/b76faad89e374ded8b9b339f36e518f3 2024-11-08T02:24:51,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c5ef8ee23909456e8389e51efb04381b is 50, key is test_row_0/C:col10/1731032690755/Put/seqid=0 2024-11-08T02:24:51,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742331_1507 (size=12301) 2024-11-08T02:24:51,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:51,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032751945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:51,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T02:24:52,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032752058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:52,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032752268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:52,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:52,326 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c5ef8ee23909456e8389e51efb04381b 2024-11-08T02:24:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/f83107bcb708450faee2433335653de4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/f83107bcb708450faee2433335653de4 2024-11-08T02:24:52,331 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/f83107bcb708450faee2433335653de4, entries=150, sequenceid=314, filesize=12.0 K 2024-11-08T02:24:52,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/b76faad89e374ded8b9b339f36e518f3 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b76faad89e374ded8b9b339f36e518f3 2024-11-08T02:24:52,334 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b76faad89e374ded8b9b339f36e518f3, entries=150, sequenceid=314, filesize=12.0 K 2024-11-08T02:24:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/c5ef8ee23909456e8389e51efb04381b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c5ef8ee23909456e8389e51efb04381b 2024-11-08T02:24:52,337 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c5ef8ee23909456e8389e51efb04381b, entries=150, sequenceid=314, filesize=12.0 K 2024-11-08T02:24:52,338 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9024be227c18091a068b12c5acebba9f in 445ms, sequenceid=314, compaction requested=true 2024-11-08T02:24:52,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:52,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:52,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-08T02:24:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-08T02:24:52,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-08T02:24:52,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 598 msec 2024-11-08T02:24:52,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 602 msec 2024-11-08T02:24:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T02:24:52,368 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:52,369 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-08T02:24:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T02:24:52,370 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:52,371 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:52,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T02:24:52,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-08T02:24:52,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:52,523 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:24:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/8e92e09df23545edaec39fb37fa32722 is 50, key is test_row_1/A:col10/1731032691944/Put/seqid=0 2024-11-08T02:24:52,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742332_1508 (size=9857) 2024-11-08T02:24:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:52,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T02:24:52,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032752684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:52,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032752797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:52,929 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/8e92e09df23545edaec39fb37fa32722 2024-11-08T02:24:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/3ed1ff43f2fb4682aa9a6c4c6165259c is 50, key is test_row_1/B:col10/1731032691944/Put/seqid=0 2024-11-08T02:24:52,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742333_1509 (size=9857) 2024-11-08T02:24:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T02:24:53,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032753008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032753317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:53,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:53,338 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/3ed1ff43f2fb4682aa9a6c4c6165259c 2024-11-08T02:24:53,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/88879f974f514d3a86e59571de03187f is 50, key is test_row_1/C:col10/1731032691944/Put/seqid=0 2024-11-08T02:24:53,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742334_1510 (size=9857) 2024-11-08T02:24:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T02:24:53,747 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/88879f974f514d3a86e59571de03187f 2024-11-08T02:24:53,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/8e92e09df23545edaec39fb37fa32722 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/8e92e09df23545edaec39fb37fa32722 2024-11-08T02:24:53,753 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/8e92e09df23545edaec39fb37fa32722, entries=100, sequenceid=325, filesize=9.6 K 2024-11-08T02:24:53,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/3ed1ff43f2fb4682aa9a6c4c6165259c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/3ed1ff43f2fb4682aa9a6c4c6165259c 2024-11-08T02:24:53,755 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/3ed1ff43f2fb4682aa9a6c4c6165259c, entries=100, sequenceid=325, filesize=9.6 K 2024-11-08T02:24:53,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/88879f974f514d3a86e59571de03187f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/88879f974f514d3a86e59571de03187f 2024-11-08T02:24:53,761 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/88879f974f514d3a86e59571de03187f, entries=100, sequenceid=325, filesize=9.6 K 2024-11-08T02:24:53,761 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9024be227c18091a068b12c5acebba9f in 1238ms, sequenceid=325, compaction requested=true 2024-11-08T02:24:53,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:53,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:53,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-08T02:24:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-08T02:24:53,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-08T02:24:53,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3910 sec 2024-11-08T02:24:53,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3950 sec 2024-11-08T02:24:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:53,831 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:24:53,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:24:53,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/cb563d9f9ea4423397518abd41d4bae7 is 50, key is test_row_0/A:col10/1731032693830/Put/seqid=0 2024-11-08T02:24:53,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742335_1511 (size=14741) 2024-11-08T02:24:53,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032753872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:53,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032753988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:54,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47104 deadline: 1731032754188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:54,193 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:54,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032754198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47136 deadline: 1731032754207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:54,212 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:54,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47088 deadline: 1731032754218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:54,225 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 9024be227c18091a068b12c5acebba9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:24:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:54,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/cb563d9f9ea4423397518abd41d4bae7 2024-11-08T02:24:54,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/230a8760b3544cb0b4479ef92a784bc9 is 50, key is test_row_0/B:col10/1731032693830/Put/seqid=0 2024-11-08T02:24:54,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742336_1512 (size=12301) 2024-11-08T02:24:54,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:24:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731032754507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 2024-11-08T02:24:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 , the old value is region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:24:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., hostname=331c7316141f,34325,1731032548147, seqNum=2 because the exception is null or not the one we care about 2024-11-08T02:24:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T02:24:54,518 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:24:54,518 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:24:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-08T02:24:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T02:24:54,520 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:24:54,520 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:24:54,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:24:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T02:24:54,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/230a8760b3544cb0b4479ef92a784bc9 2024-11-08T02:24:54,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/8fef09d855564dd5a4ae89b124039cdd is 50, key is test_row_0/C:col10/1731032693830/Put/seqid=0 2024-11-08T02:24:54,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742337_1513 (size=12301) 2024-11-08T02:24:54,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/8fef09d855564dd5a4ae89b124039cdd 2024-11-08T02:24:54,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/cb563d9f9ea4423397518abd41d4bae7 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cb563d9f9ea4423397518abd41d4bae7 2024-11-08T02:24:54,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-08T02:24:54,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:54,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:54,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:54,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:54,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:24:54,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cb563d9f9ea4423397518abd41d4bae7, entries=200, sequenceid=351, filesize=14.4 K 2024-11-08T02:24:54,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/230a8760b3544cb0b4479ef92a784bc9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/230a8760b3544cb0b4479ef92a784bc9 2024-11-08T02:24:54,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/230a8760b3544cb0b4479ef92a784bc9, entries=150, sequenceid=351, filesize=12.0 K 2024-11-08T02:24:54,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/8fef09d855564dd5a4ae89b124039cdd as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8fef09d855564dd5a4ae89b124039cdd 2024-11-08T02:24:54,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8fef09d855564dd5a4ae89b124039cdd, entries=150, sequenceid=351, filesize=12.0 K 2024-11-08T02:24:54,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 9024be227c18091a068b12c5acebba9f in 851ms, sequenceid=351, compaction requested=true 2024-11-08T02:24:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:24:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:54,682 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T02:24:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:24:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:54,683 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T02:24:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9024be227c18091a068b12c5acebba9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:24:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:54,684 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64589 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T02:24:54,684 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59709 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T02:24:54,684 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/A is initiating minor compaction (all files) 2024-11-08T02:24:54,684 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/B is initiating minor compaction (all files) 2024-11-08T02:24:54,684 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/A in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:54,684 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/B in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:54,684 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/27dea432956b40e39612368df8ea23a6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/e77297b378d4408186be828ad44c7e1d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/f83107bcb708450faee2433335653de4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/8e92e09df23545edaec39fb37fa32722, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cb563d9f9ea4423397518abd41d4bae7] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=63.1 K 2024-11-08T02:24:54,684 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/05d59e80bba04e31b10444b4a511b08f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/723244f8634e46cc96f00a0bf5be0922, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b76faad89e374ded8b9b339f36e518f3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/3ed1ff43f2fb4682aa9a6c4c6165259c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/230a8760b3544cb0b4479ef92a784bc9] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=58.3 K 2024-11-08T02:24:54,684 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27dea432956b40e39612368df8ea23a6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731032689364 2024-11-08T02:24:54,684 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 05d59e80bba04e31b10444b4a511b08f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731032689364 2024-11-08T02:24:54,686 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting e77297b378d4408186be828ad44c7e1d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731032690659 2024-11-08T02:24:54,686 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 723244f8634e46cc96f00a0bf5be0922, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731032690659 2024-11-08T02:24:54,686 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting b76faad89e374ded8b9b339f36e518f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731032690755 2024-11-08T02:24:54,686 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting f83107bcb708450faee2433335653de4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731032690755 2024-11-08T02:24:54,686 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ed1ff43f2fb4682aa9a6c4c6165259c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731032691938 2024-11-08T02:24:54,686 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e92e09df23545edaec39fb37fa32722, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731032691938 2024-11-08T02:24:54,687 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 230a8760b3544cb0b4479ef92a784bc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731032692647 2024-11-08T02:24:54,687 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb563d9f9ea4423397518abd41d4bae7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731032692647 2024-11-08T02:24:54,700 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#B#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:54,701 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#A#compaction#442 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:54,701 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/2062899ed53a476fabbc063fa447ec8a is 50, key is test_row_0/B:col10/1731032693830/Put/seqid=0 2024-11-08T02:24:54,701 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/b5e2af3f9d2c48abb640a34f789c0902 is 50, key is test_row_0/A:col10/1731032693830/Put/seqid=0 2024-11-08T02:24:54,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742339_1515 (size=13119) 2024-11-08T02:24:54,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742338_1514 (size=13119) 2024-11-08T02:24:54,823 INFO [Thread-1994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1994. 2024-11-08T02:24:54,823 DEBUG [Thread-1994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:54,823 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:54,824 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:54,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:54,825 INFO [Thread-1996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1996. 2024-11-08T02:24:54,825 DEBUG [Thread-1996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:54,825 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:54,825 DEBUG [Thread-1996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:54,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:54,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-08T02:24:54,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:54,826 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T02:24:54,826 INFO [Thread-1992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1992. 2024-11-08T02:24:54,826 DEBUG [Thread-1992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:54,826 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:54,826 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:54,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:54,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/fe390c1b592347a2b2bf978f4fcc1968 is 50, key is test_row_0/A:col10/1731032693870/Put/seqid=0 2024-11-08T02:24:54,832 INFO [Thread-2000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2000. 2024-11-08T02:24:54,832 INFO [Thread-1998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1998. 2024-11-08T02:24:54,832 DEBUG [Thread-1998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:54,832 DEBUG [Thread-2000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:54,832 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:54,832 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:54,832 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:54,832 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:54,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:54,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:54,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742340_1516 (size=9857) 2024-11-08T02:24:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T02:24:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 9024be227c18091a068b12c5acebba9f 2024-11-08T02:24:55,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. as already flushing 2024-11-08T02:24:55,022 INFO [Thread-1983 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1983. 2024-11-08T02:24:55,022 DEBUG [Thread-1983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:55,022 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:55,022 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:55,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:55,118 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/2062899ed53a476fabbc063fa447ec8a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/2062899ed53a476fabbc063fa447ec8a 2024-11-08T02:24:55,122 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/b5e2af3f9d2c48abb640a34f789c0902 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/b5e2af3f9d2c48abb640a34f789c0902 2024-11-08T02:24:55,123 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9024be227c18091a068b12c5acebba9f/B of 9024be227c18091a068b12c5acebba9f into 2062899ed53a476fabbc063fa447ec8a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:55,123 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:55,123 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/B, priority=11, startTime=1731032694682; duration=0sec 2024-11-08T02:24:55,123 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:24:55,123 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:B 2024-11-08T02:24:55,123 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T02:24:55,124 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59709 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T02:24:55,124 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 9024be227c18091a068b12c5acebba9f/C is initiating minor compaction (all files) 2024-11-08T02:24:55,124 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9024be227c18091a068b12c5acebba9f/C in TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:55,124 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/f2f785a411c74b09958df5b246dfa066, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/f00e92cd5c724f5f9258aec86b104b03, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c5ef8ee23909456e8389e51efb04381b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/88879f974f514d3a86e59571de03187f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8fef09d855564dd5a4ae89b124039cdd] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp, totalSize=58.3 K 2024-11-08T02:24:55,125 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f785a411c74b09958df5b246dfa066, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731032689364 2024-11-08T02:24:55,125 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting f00e92cd5c724f5f9258aec86b104b03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731032690659 2024-11-08T02:24:55,125 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ef8ee23909456e8389e51efb04381b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731032690755 2024-11-08T02:24:55,126 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 88879f974f514d3a86e59571de03187f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731032691938 2024-11-08T02:24:55,126 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fef09d855564dd5a4ae89b124039cdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731032692647 2024-11-08T02:24:55,126 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9024be227c18091a068b12c5acebba9f/A of 9024be227c18091a068b12c5acebba9f into b5e2af3f9d2c48abb640a34f789c0902(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:55,126 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:55,127 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/A, priority=11, startTime=1731032694682; duration=0sec 2024-11-08T02:24:55,127 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:55,127 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:A 2024-11-08T02:24:55,135 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9024be227c18091a068b12c5acebba9f#C#compaction#444 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:24:55,135 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/60b3fb634c7a46c8bf9ed85a8973030c is 50, key is test_row_0/C:col10/1731032693830/Put/seqid=0 2024-11-08T02:24:55,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742341_1517 (size=13119) 2024-11-08T02:24:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T02:24:55,235 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/fe390c1b592347a2b2bf978f4fcc1968 2024-11-08T02:24:55,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/d742160fa7a045c2890e41fe468d8b5b is 50, key is test_row_0/B:col10/1731032693870/Put/seqid=0 2024-11-08T02:24:55,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742342_1518 (size=9857) 2024-11-08T02:24:55,550 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/60b3fb634c7a46c8bf9ed85a8973030c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/60b3fb634c7a46c8bf9ed85a8973030c 2024-11-08T02:24:55,557 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 9024be227c18091a068b12c5acebba9f/C of 9024be227c18091a068b12c5acebba9f into 60b3fb634c7a46c8bf9ed85a8973030c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:24:55,557 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:55,557 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., storeName=9024be227c18091a068b12c5acebba9f/C, priority=11, startTime=1731032694683; duration=0sec 2024-11-08T02:24:55,557 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:24:55,557 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9024be227c18091a068b12c5acebba9f:C 2024-11-08T02:24:55,644 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/d742160fa7a045c2890e41fe468d8b5b 2024-11-08T02:24:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T02:24:55,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/5b2547c792cd498981acd337147bb191 is 50, key is test_row_0/C:col10/1731032693870/Put/seqid=0 2024-11-08T02:24:55,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742343_1519 (size=9857) 2024-11-08T02:24:56,065 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/5b2547c792cd498981acd337147bb191 2024-11-08T02:24:56,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/fe390c1b592347a2b2bf978f4fcc1968 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/fe390c1b592347a2b2bf978f4fcc1968 2024-11-08T02:24:56,078 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/fe390c1b592347a2b2bf978f4fcc1968, entries=100, sequenceid=361, filesize=9.6 K 2024-11-08T02:24:56,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/d742160fa7a045c2890e41fe468d8b5b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/d742160fa7a045c2890e41fe468d8b5b 2024-11-08T02:24:56,082 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/d742160fa7a045c2890e41fe468d8b5b, entries=100, sequenceid=361, filesize=9.6 K 2024-11-08T02:24:56,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/5b2547c792cd498981acd337147bb191 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/5b2547c792cd498981acd337147bb191 2024-11-08T02:24:56,086 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/5b2547c792cd498981acd337147bb191, entries=100, sequenceid=361, filesize=9.6 K 2024-11-08T02:24:56,086 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=6.71 KB/6870 for 9024be227c18091a068b12c5acebba9f in 1260ms, sequenceid=361, compaction requested=false 2024-11-08T02:24:56,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 9024be227c18091a068b12c5acebba9f: 2024-11-08T02:24:56,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:24:56,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-08T02:24:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-08T02:24:56,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-08T02:24:56,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5680 sec 2024-11-08T02:24:56,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.5740 sec 2024-11-08T02:24:56,570 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:24:56,632 INFO [Thread-1989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1989. 2024-11-08T02:24:56,633 DEBUG [Thread-1989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:24:56,633 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:56,633 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:24:56,633 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:24:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T02:24:56,669 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:04,229 INFO [Thread-1987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1987. 2024-11-08T02:25:04,229 INFO [Thread-1985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1985. 2024-11-08T02:25:04,230 DEBUG [Thread-1987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:04,230 DEBUG [Thread-1985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:04,230 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:04,230 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:04,230 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:04,230 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:04,230 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:04,230 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:04,249 INFO [Thread-1981 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1981. 2024-11-08T02:25:04,249 DEBUG [Thread-1981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:04,249 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:04,249 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:04,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 172 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1808 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5416 rows 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1809 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5427 rows 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1804 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5404 rows 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1812 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5435 rows 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1813 2024-11-08T02:25:04,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5433 rows 2024-11-08T02:25:04,249 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T02:25:04,249 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:25:04,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:04,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:04,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:04,250 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:25:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T02:25:04,252 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032704252"}]},"ts":"1731032704252"} 2024-11-08T02:25:04,253 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:25:04,253 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:25:04,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:25:04,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9024be227c18091a068b12c5acebba9f, UNASSIGN}] 2024-11-08T02:25:04,255 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9024be227c18091a068b12c5acebba9f, UNASSIGN 2024-11-08T02:25:04,255 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=9024be227c18091a068b12c5acebba9f, regionState=CLOSING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:25:04,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9024be227c18091a068b12c5acebba9f, UNASSIGN because future has completed 2024-11-08T02:25:04,256 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:25:04,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:25:04,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T02:25:04,408 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 9024be227c18091a068b12c5acebba9f 2024-11-08T02:25:04,408 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T02:25:04,408 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 9024be227c18091a068b12c5acebba9f, disabling compactions & flushes 2024-11-08T02:25:04,408 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:25:04,408 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:25:04,408 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. after waiting 0 ms 2024-11-08T02:25:04,408 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:25:04,409 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 9024be227c18091a068b12c5acebba9f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T02:25:04,411 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/143592e1d2c64eee984a893a49ff0472 is 50, key is test_row_1/A:col10/1731032704248/Put/seqid=0 2024-11-08T02:25:04,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742344_1520 (size=9857) 2024-11-08T02:25:04,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T02:25:04,815 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/143592e1d2c64eee984a893a49ff0472 2024-11-08T02:25:04,820 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/b37f4e13c5d247ff8a0b281c1915564b is 50, key is test_row_1/B:col10/1731032704248/Put/seqid=0 2024-11-08T02:25:04,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742345_1521 (size=9857) 2024-11-08T02:25:04,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T02:25:05,223 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/b37f4e13c5d247ff8a0b281c1915564b 2024-11-08T02:25:05,228 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/54a4f1f0f7834c209d5a07bf7bc22327 is 50, key is test_row_1/C:col10/1731032704248/Put/seqid=0 2024-11-08T02:25:05,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742346_1522 (size=9857) 2024-11-08T02:25:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T02:25:05,631 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/54a4f1f0f7834c209d5a07bf7bc22327 2024-11-08T02:25:05,634 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/A/143592e1d2c64eee984a893a49ff0472 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/143592e1d2c64eee984a893a49ff0472 2024-11-08T02:25:05,637 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/143592e1d2c64eee984a893a49ff0472, entries=100, sequenceid=372, filesize=9.6 K 2024-11-08T02:25:05,637 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/B/b37f4e13c5d247ff8a0b281c1915564b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b37f4e13c5d247ff8a0b281c1915564b 2024-11-08T02:25:05,639 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b37f4e13c5d247ff8a0b281c1915564b, entries=100, sequenceid=372, filesize=9.6 K 2024-11-08T02:25:05,640 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/.tmp/C/54a4f1f0f7834c209d5a07bf7bc22327 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/54a4f1f0f7834c209d5a07bf7bc22327 2024-11-08T02:25:05,642 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/54a4f1f0f7834c209d5a07bf7bc22327, entries=100, sequenceid=372, filesize=9.6 K 2024-11-08T02:25:05,643 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9024be227c18091a068b12c5acebba9f in 1235ms, sequenceid=372, compaction requested=true 2024-11-08T02:25:05,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/77fed8fd945c4684a36da477c51de08e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/11ce8f180da4495cb0cec22c357282c5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/6d3f234c9ffb4bbb8c48e31e85395d7c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/ed4c3b3f20df469d969f4a9a6a17a6f3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/1bd28cc941954ccc8df85a607f0ce44e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/63a10216b71946818045a9fdce7e558a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/81ef2e7542104904bfd1c57b9f9f6f7d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/62b92e61a18d4551b94af4741694877f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/75eb4f78c42a416c897265205bfaa77c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/5d11697cf0794a18a8b3e5e339081c84, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/938b011f004c445a9dffa587ad4f5cdc, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/0b04b00f39cf430d9f61da7eef5b1088, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/62fb900099b54417bdc2addfb2a24c88, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/a6ea7c7727894565a1e6b3009a143c8e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/d3e8d58e2ca54e53acad8e8780f3bce5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/4500ec57c1bb48c1b5821c31f77a54d5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/c610a65e07f4446c80bb78647691a769, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cdb78eac0dae4f199be3bb1fe0e91f34, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/3926df139b39470dbdd6125d72833e10, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/27dea432956b40e39612368df8ea23a6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/e77297b378d4408186be828ad44c7e1d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/f83107bcb708450faee2433335653de4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/8e92e09df23545edaec39fb37fa32722, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cb563d9f9ea4423397518abd41d4bae7] to archive 2024-11-08T02:25:05,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:25:05,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/77fed8fd945c4684a36da477c51de08e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/77fed8fd945c4684a36da477c51de08e 2024-11-08T02:25:05,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/11ce8f180da4495cb0cec22c357282c5 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/11ce8f180da4495cb0cec22c357282c5 2024-11-08T02:25:05,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/6d3f234c9ffb4bbb8c48e31e85395d7c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/6d3f234c9ffb4bbb8c48e31e85395d7c 2024-11-08T02:25:05,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/ed4c3b3f20df469d969f4a9a6a17a6f3 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/ed4c3b3f20df469d969f4a9a6a17a6f3 2024-11-08T02:25:05,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/1bd28cc941954ccc8df85a607f0ce44e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/1bd28cc941954ccc8df85a607f0ce44e 2024-11-08T02:25:05,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/63a10216b71946818045a9fdce7e558a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/63a10216b71946818045a9fdce7e558a 2024-11-08T02:25:05,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/81ef2e7542104904bfd1c57b9f9f6f7d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/81ef2e7542104904bfd1c57b9f9f6f7d 2024-11-08T02:25:05,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/62b92e61a18d4551b94af4741694877f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/62b92e61a18d4551b94af4741694877f 2024-11-08T02:25:05,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/75eb4f78c42a416c897265205bfaa77c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/75eb4f78c42a416c897265205bfaa77c 2024-11-08T02:25:05,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/5d11697cf0794a18a8b3e5e339081c84 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/5d11697cf0794a18a8b3e5e339081c84 2024-11-08T02:25:05,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/938b011f004c445a9dffa587ad4f5cdc to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/938b011f004c445a9dffa587ad4f5cdc 2024-11-08T02:25:05,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/0b04b00f39cf430d9f61da7eef5b1088 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/0b04b00f39cf430d9f61da7eef5b1088 2024-11-08T02:25:05,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/62fb900099b54417bdc2addfb2a24c88 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/62fb900099b54417bdc2addfb2a24c88 2024-11-08T02:25:05,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/a6ea7c7727894565a1e6b3009a143c8e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/a6ea7c7727894565a1e6b3009a143c8e 2024-11-08T02:25:05,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/d3e8d58e2ca54e53acad8e8780f3bce5 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/d3e8d58e2ca54e53acad8e8780f3bce5 2024-11-08T02:25:05,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/4500ec57c1bb48c1b5821c31f77a54d5 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/4500ec57c1bb48c1b5821c31f77a54d5 2024-11-08T02:25:05,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/c610a65e07f4446c80bb78647691a769 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/c610a65e07f4446c80bb78647691a769 2024-11-08T02:25:05,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cdb78eac0dae4f199be3bb1fe0e91f34 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cdb78eac0dae4f199be3bb1fe0e91f34 2024-11-08T02:25:05,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/3926df139b39470dbdd6125d72833e10 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/3926df139b39470dbdd6125d72833e10 2024-11-08T02:25:05,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/27dea432956b40e39612368df8ea23a6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/27dea432956b40e39612368df8ea23a6 2024-11-08T02:25:05,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/e77297b378d4408186be828ad44c7e1d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/e77297b378d4408186be828ad44c7e1d 2024-11-08T02:25:05,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/f83107bcb708450faee2433335653de4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/f83107bcb708450faee2433335653de4 2024-11-08T02:25:05,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/8e92e09df23545edaec39fb37fa32722 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/8e92e09df23545edaec39fb37fa32722 2024-11-08T02:25:05,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cb563d9f9ea4423397518abd41d4bae7 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/cb563d9f9ea4423397518abd41d4bae7 2024-11-08T02:25:05,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/23bd6c1e4ab24503976ff169ecdff32a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/696abaeac09246aebb81442ac210148d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/9ea19cc57a0147eb9410d23c6f8257ba, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/8067695e3c5b4376832b6a1ab6c477e1, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/89555b4ef3c8481e98e0e1dd8b996f77, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/1aa5bf460a7941dc8799eba0907ab284, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/ba0ef423fd664345af427ac8e4a45adb, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/7627f957b9bf4ce99b6ff936fbcab975, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/4cf6cb704aaf4543bce4b63b66554997, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/a5d1bf5b487543f6a28d902f4784a26f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/bb839d6423c1429c976134c12e37f9c0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/91c09257964a474f942f83d7646b0057, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/37c315ff9e654602aa6434a42e7a07ae, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/dc86efb10d094b65aebbe52101fb4962, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/edaa0bfb091c4367a84ac61b69f33d05, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b85dd5cb27e945fbbc416f3915025ad8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/2106676296e44647b5dbaaff4346343a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b7e5ddd61b494c3aa9028286722df985, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/05d59e80bba04e31b10444b4a511b08f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/cc2482624d204ec0a305d8bb22bb26d7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/723244f8634e46cc96f00a0bf5be0922, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b76faad89e374ded8b9b339f36e518f3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/3ed1ff43f2fb4682aa9a6c4c6165259c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/230a8760b3544cb0b4479ef92a784bc9] to archive 2024-11-08T02:25:05,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:25:05,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/23bd6c1e4ab24503976ff169ecdff32a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/23bd6c1e4ab24503976ff169ecdff32a 2024-11-08T02:25:05,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/696abaeac09246aebb81442ac210148d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/696abaeac09246aebb81442ac210148d 2024-11-08T02:25:05,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/9ea19cc57a0147eb9410d23c6f8257ba to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/9ea19cc57a0147eb9410d23c6f8257ba 2024-11-08T02:25:05,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/8067695e3c5b4376832b6a1ab6c477e1 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/8067695e3c5b4376832b6a1ab6c477e1 2024-11-08T02:25:05,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/89555b4ef3c8481e98e0e1dd8b996f77 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/89555b4ef3c8481e98e0e1dd8b996f77 2024-11-08T02:25:05,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/1aa5bf460a7941dc8799eba0907ab284 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/1aa5bf460a7941dc8799eba0907ab284 2024-11-08T02:25:05,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/ba0ef423fd664345af427ac8e4a45adb to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/ba0ef423fd664345af427ac8e4a45adb 2024-11-08T02:25:05,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/7627f957b9bf4ce99b6ff936fbcab975 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/7627f957b9bf4ce99b6ff936fbcab975 2024-11-08T02:25:05,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/4cf6cb704aaf4543bce4b63b66554997 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/4cf6cb704aaf4543bce4b63b66554997 2024-11-08T02:25:05,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/a5d1bf5b487543f6a28d902f4784a26f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/a5d1bf5b487543f6a28d902f4784a26f 2024-11-08T02:25:05,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/bb839d6423c1429c976134c12e37f9c0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/bb839d6423c1429c976134c12e37f9c0 2024-11-08T02:25:05,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/91c09257964a474f942f83d7646b0057 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/91c09257964a474f942f83d7646b0057 2024-11-08T02:25:05,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/37c315ff9e654602aa6434a42e7a07ae to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/37c315ff9e654602aa6434a42e7a07ae 2024-11-08T02:25:05,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/dc86efb10d094b65aebbe52101fb4962 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/dc86efb10d094b65aebbe52101fb4962 2024-11-08T02:25:05,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/edaa0bfb091c4367a84ac61b69f33d05 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/edaa0bfb091c4367a84ac61b69f33d05 2024-11-08T02:25:05,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b85dd5cb27e945fbbc416f3915025ad8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b85dd5cb27e945fbbc416f3915025ad8 2024-11-08T02:25:05,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/2106676296e44647b5dbaaff4346343a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/2106676296e44647b5dbaaff4346343a 2024-11-08T02:25:05,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b7e5ddd61b494c3aa9028286722df985 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b7e5ddd61b494c3aa9028286722df985 2024-11-08T02:25:05,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/05d59e80bba04e31b10444b4a511b08f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/05d59e80bba04e31b10444b4a511b08f 2024-11-08T02:25:05,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/cc2482624d204ec0a305d8bb22bb26d7 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/cc2482624d204ec0a305d8bb22bb26d7 2024-11-08T02:25:05,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/723244f8634e46cc96f00a0bf5be0922 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/723244f8634e46cc96f00a0bf5be0922 2024-11-08T02:25:05,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b76faad89e374ded8b9b339f36e518f3 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b76faad89e374ded8b9b339f36e518f3 2024-11-08T02:25:05,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/3ed1ff43f2fb4682aa9a6c4c6165259c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/3ed1ff43f2fb4682aa9a6c4c6165259c 2024-11-08T02:25:05,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/230a8760b3544cb0b4479ef92a784bc9 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/230a8760b3544cb0b4479ef92a784bc9 2024-11-08T02:25:05,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/d9806859a91f45bb8332b33944877576, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/b4f1db6e035349e884c528179a5da868, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/bb9d55ca77ec4ee59d55ab0d4971b328, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c4cdd0c9a7904957aec6aa39ce31001a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c83fece32e364ed79d9b30661c63fcc0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2ce696d3cb664845a2dd722ade88738c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2da01661256046f7aad5179a0ea80560, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/a1bd551e2c6c44d187176a42cdda50c2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/1558335d8a5949ea9049a8402c8e8f0a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/6bcdbc5b0f824558ad65b334a68b7326, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c26f1c396e62449cb86b404a28bb96bd, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8b46f2a6941540f68cbe0ea099186ae7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/3d8b8a5c8c7c48088ede857aadfe4a19, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/55d68927babc49c292b057a69841d33a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c6f220924b124f72bb06fbd8f7017754, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c84a3d5509fc4f14a07364089d61fb3d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/601dc582a67346cc89012c9b3d998c11, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/5edb88e70b6e4f79ad3ffa6726d001e0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/f2f785a411c74b09958df5b246dfa066, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2c00f682a203447d868f1ff02eab0c76, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/f00e92cd5c724f5f9258aec86b104b03, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c5ef8ee23909456e8389e51efb04381b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/88879f974f514d3a86e59571de03187f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8fef09d855564dd5a4ae89b124039cdd] to archive 2024-11-08T02:25:05,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:25:05,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/d9806859a91f45bb8332b33944877576 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/d9806859a91f45bb8332b33944877576 2024-11-08T02:25:05,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/b4f1db6e035349e884c528179a5da868 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/b4f1db6e035349e884c528179a5da868 2024-11-08T02:25:05,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/bb9d55ca77ec4ee59d55ab0d4971b328 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/bb9d55ca77ec4ee59d55ab0d4971b328 2024-11-08T02:25:05,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c4cdd0c9a7904957aec6aa39ce31001a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c4cdd0c9a7904957aec6aa39ce31001a 2024-11-08T02:25:05,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c83fece32e364ed79d9b30661c63fcc0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c83fece32e364ed79d9b30661c63fcc0 2024-11-08T02:25:05,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2ce696d3cb664845a2dd722ade88738c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2ce696d3cb664845a2dd722ade88738c 2024-11-08T02:25:05,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2da01661256046f7aad5179a0ea80560 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2da01661256046f7aad5179a0ea80560 2024-11-08T02:25:05,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/a1bd551e2c6c44d187176a42cdda50c2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/a1bd551e2c6c44d187176a42cdda50c2 2024-11-08T02:25:05,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/1558335d8a5949ea9049a8402c8e8f0a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/1558335d8a5949ea9049a8402c8e8f0a 2024-11-08T02:25:05,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/6bcdbc5b0f824558ad65b334a68b7326 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/6bcdbc5b0f824558ad65b334a68b7326 2024-11-08T02:25:05,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c26f1c396e62449cb86b404a28bb96bd to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c26f1c396e62449cb86b404a28bb96bd 2024-11-08T02:25:05,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8b46f2a6941540f68cbe0ea099186ae7 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8b46f2a6941540f68cbe0ea099186ae7 2024-11-08T02:25:05,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/3d8b8a5c8c7c48088ede857aadfe4a19 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/3d8b8a5c8c7c48088ede857aadfe4a19 2024-11-08T02:25:05,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/55d68927babc49c292b057a69841d33a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/55d68927babc49c292b057a69841d33a 2024-11-08T02:25:05,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c6f220924b124f72bb06fbd8f7017754 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c6f220924b124f72bb06fbd8f7017754 2024-11-08T02:25:05,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c84a3d5509fc4f14a07364089d61fb3d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c84a3d5509fc4f14a07364089d61fb3d 2024-11-08T02:25:05,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/601dc582a67346cc89012c9b3d998c11 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/601dc582a67346cc89012c9b3d998c11 2024-11-08T02:25:05,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/5edb88e70b6e4f79ad3ffa6726d001e0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/5edb88e70b6e4f79ad3ffa6726d001e0 2024-11-08T02:25:05,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/f2f785a411c74b09958df5b246dfa066 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/f2f785a411c74b09958df5b246dfa066 2024-11-08T02:25:05,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2c00f682a203447d868f1ff02eab0c76 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/2c00f682a203447d868f1ff02eab0c76 2024-11-08T02:25:05,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/f00e92cd5c724f5f9258aec86b104b03 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/f00e92cd5c724f5f9258aec86b104b03 2024-11-08T02:25:05,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c5ef8ee23909456e8389e51efb04381b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/c5ef8ee23909456e8389e51efb04381b 2024-11-08T02:25:05,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/88879f974f514d3a86e59571de03187f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/88879f974f514d3a86e59571de03187f 2024-11-08T02:25:05,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8fef09d855564dd5a4ae89b124039cdd to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/8fef09d855564dd5a4ae89b124039cdd 2024-11-08T02:25:05,698 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/recovered.edits/375.seqid, newMaxSeqId=375, maxSeqId=1 2024-11-08T02:25:05,699 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. 2024-11-08T02:25:05,699 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 9024be227c18091a068b12c5acebba9f: Waiting for close lock at 1731032704408Running coprocessor pre-close hooks at 1731032704408Disabling compacts and flushes for region at 1731032704408Disabling writes for close at 1731032704408Obtaining lock to block concurrent updates at 1731032704409 (+1 ms)Preparing flush snapshotting stores in 9024be227c18091a068b12c5acebba9f at 1731032704409Finished memstore snapshotting TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731032704409Flushing stores of TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f. at 1731032704409Flushing 9024be227c18091a068b12c5acebba9f/A: creating writer at 1731032704409Flushing 9024be227c18091a068b12c5acebba9f/A: appending metadata at 1731032704411 (+2 ms)Flushing 9024be227c18091a068b12c5acebba9f/A: closing flushed file at 1731032704411Flushing 9024be227c18091a068b12c5acebba9f/B: creating writer at 1731032704818 (+407 ms)Flushing 9024be227c18091a068b12c5acebba9f/B: appending metadata at 1731032704819 (+1 ms)Flushing 9024be227c18091a068b12c5acebba9f/B: closing flushed file at 1731032704819Flushing 9024be227c18091a068b12c5acebba9f/C: creating writer at 1731032705226 (+407 ms)Flushing 9024be227c18091a068b12c5acebba9f/C: appending metadata at 1731032705228 (+2 ms)Flushing 9024be227c18091a068b12c5acebba9f/C: closing flushed file at 1731032705228Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42421764: reopening flushed file at 1731032705634 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68acb74a: reopening flushed file at 1731032705637 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a7ed4f2: reopening flushed file at 1731032705640 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9024be227c18091a068b12c5acebba9f in 1235ms, sequenceid=372, compaction requested=true at 1731032705643 (+3 ms)Writing region close event to WAL at 1731032705696 (+53 ms)Running coprocessor post-close hooks at 1731032705699 (+3 ms)Closed at 1731032705699 2024-11-08T02:25:05,701 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 9024be227c18091a068b12c5acebba9f 2024-11-08T02:25:05,701 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=9024be227c18091a068b12c5acebba9f, regionState=CLOSED 2024-11-08T02:25:05,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:25:05,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-08T02:25:05,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 9024be227c18091a068b12c5acebba9f, server=331c7316141f,34325,1731032548147 in 1.4470 sec 2024-11-08T02:25:05,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-08T02:25:05,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9024be227c18091a068b12c5acebba9f, UNASSIGN in 1.4500 sec 2024-11-08T02:25:05,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-08T02:25:05,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4530 sec 2024-11-08T02:25:05,708 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032705708"}]},"ts":"1731032705708"} 2024-11-08T02:25:05,709 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:25:05,709 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:25:05,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4600 sec 2024-11-08T02:25:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T02:25:06,398 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:06,399 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:25:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:06,400 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-08T02:25:06,400 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:06,402 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f 2024-11-08T02:25:06,403 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/recovered.edits] 2024-11-08T02:25:06,405 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/143592e1d2c64eee984a893a49ff0472 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/143592e1d2c64eee984a893a49ff0472 2024-11-08T02:25:06,406 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/b5e2af3f9d2c48abb640a34f789c0902 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/b5e2af3f9d2c48abb640a34f789c0902 2024-11-08T02:25:06,406 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/fe390c1b592347a2b2bf978f4fcc1968 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/A/fe390c1b592347a2b2bf978f4fcc1968 2024-11-08T02:25:06,408 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/2062899ed53a476fabbc063fa447ec8a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/2062899ed53a476fabbc063fa447ec8a 2024-11-08T02:25:06,409 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b37f4e13c5d247ff8a0b281c1915564b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/b37f4e13c5d247ff8a0b281c1915564b 2024-11-08T02:25:06,409 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/d742160fa7a045c2890e41fe468d8b5b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/B/d742160fa7a045c2890e41fe468d8b5b 2024-11-08T02:25:06,411 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/54a4f1f0f7834c209d5a07bf7bc22327 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/54a4f1f0f7834c209d5a07bf7bc22327 2024-11-08T02:25:06,411 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/5b2547c792cd498981acd337147bb191 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/5b2547c792cd498981acd337147bb191 2024-11-08T02:25:06,412 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/60b3fb634c7a46c8bf9ed85a8973030c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/C/60b3fb634c7a46c8bf9ed85a8973030c 2024-11-08T02:25:06,414 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/recovered.edits/375.seqid to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f/recovered.edits/375.seqid 2024-11-08T02:25:06,414 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/9024be227c18091a068b12c5acebba9f 2024-11-08T02:25:06,414 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:25:06,416 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:06,418 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:25:06,419 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:25:06,420 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:06,420 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:25:06,420 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731032706420"}]},"ts":"9223372036854775807"} 2024-11-08T02:25:06,421 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T02:25:06,421 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9024be227c18091a068b12c5acebba9f, NAME => 'TestAcidGuarantees,,1731032672604.9024be227c18091a068b12c5acebba9f.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:25:06,421 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:25:06,422 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731032706422"}]},"ts":"9223372036854775807"} 2024-11-08T02:25:06,423 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T02:25:06,423 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:06,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 25 msec 2024-11-08T02:25:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-08T02:25:06,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T02:25:06,508 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:06,515 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=236 (was 242), OpenFileDescriptor=441 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=425 (was 512), ProcessCount=11 (was 11), AvailableMemoryMB=6580 (was 6650) 2024-11-08T02:25:06,523 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=425, ProcessCount=11, AvailableMemoryMB=6580 2024-11-08T02:25:06,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:25:06,525 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:25:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:06,526 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:25:06,526 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:06,527 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-08T02:25:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T02:25:06,527 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:25:06,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742347_1523 (size=959) 2024-11-08T02:25:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T02:25:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T02:25:06,933 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee 2024-11-08T02:25:06,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742348_1524 (size=53) 2024-11-08T02:25:07,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T02:25:07,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:25:07,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7fa5796b2a900c52ba2e3fc96ec46b12, disabling compactions & flushes 2024-11-08T02:25:07,337 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:07,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:07,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. after waiting 0 ms 2024-11-08T02:25:07,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:07,337 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:07,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7fa5796b2a900c52ba2e3fc96ec46b12: Waiting for close lock at 1731032707337Disabling compacts and flushes for region at 1731032707337Disabling writes for close at 1731032707337Writing region close event to WAL at 1731032707337Closed at 1731032707337 2024-11-08T02:25:07,338 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:25:07,339 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731032707338"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731032707338"}]},"ts":"1731032707338"} 2024-11-08T02:25:07,340 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T02:25:07,340 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:25:07,340 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032707340"}]},"ts":"1731032707340"} 2024-11-08T02:25:07,341 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:25:07,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, ASSIGN}] 2024-11-08T02:25:07,342 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, ASSIGN 2024-11-08T02:25:07,343 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, ASSIGN; state=OFFLINE, location=331c7316141f,34325,1731032548147; forceNewPlan=false, retain=false 2024-11-08T02:25:07,493 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=7fa5796b2a900c52ba2e3fc96ec46b12, regionState=OPENING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:25:07,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, ASSIGN because future has completed 2024-11-08T02:25:07,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:25:07,649 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:07,649 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:25:07,649 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,649 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:25:07,649 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,649 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,650 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,651 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fa5796b2a900c52ba2e3fc96ec46b12 columnFamilyName A 2024-11-08T02:25:07,651 DEBUG [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:07,651 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(327): Store=7fa5796b2a900c52ba2e3fc96ec46b12/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:25:07,651 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,652 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fa5796b2a900c52ba2e3fc96ec46b12 columnFamilyName B 2024-11-08T02:25:07,652 DEBUG [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:07,652 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(327): Store=7fa5796b2a900c52ba2e3fc96ec46b12/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:25:07,653 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,653 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fa5796b2a900c52ba2e3fc96ec46b12 columnFamilyName C 2024-11-08T02:25:07,653 DEBUG [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:07,653 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(327): Store=7fa5796b2a900c52ba2e3fc96ec46b12/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:25:07,653 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,654 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,654 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,655 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,655 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,655 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:25:07,656 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,657 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:25:07,657 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 7fa5796b2a900c52ba2e3fc96ec46b12; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72438306, jitterRate=0.07941487431526184}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:25:07,657 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:07,658 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 7fa5796b2a900c52ba2e3fc96ec46b12: Running coprocessor pre-open hook at 1731032707649Writing region info on filesystem at 1731032707649Initializing all the Stores at 1731032707650 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032707650Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032707650Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032707650Cleaning up temporary data from old regions at 1731032707655 (+5 ms)Running coprocessor post-open hooks at 1731032707657 (+2 ms)Region opened successfully at 1731032707658 (+1 ms) 2024-11-08T02:25:07,658 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., pid=144, masterSystemTime=1731032707647 2024-11-08T02:25:07,660 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:07,660 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:07,660 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=7fa5796b2a900c52ba2e3fc96ec46b12, regionState=OPEN, openSeqNum=2, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:25:07,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:25:07,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-08T02:25:07,663 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 in 167 msec 2024-11-08T02:25:07,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-08T02:25:07,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, ASSIGN in 321 msec 2024-11-08T02:25:07,665 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:25:07,665 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032707665"}]},"ts":"1731032707665"} 2024-11-08T02:25:07,666 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:25:07,666 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:25:07,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1410 sec 2024-11-08T02:25:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-08T02:25:07,668 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:07,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:07,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:25:07,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:25:07,670 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:25:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:25:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:25:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73240d0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:25:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:25:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:07,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:25:07,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69a35090, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:07,672 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:25:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:25:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:25:07,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:25:07,674 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T02:25:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 331c7316141f,41605,1731032547397 2024-11-08T02:25:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@597e12db 2024-11-08T02:25:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:25:07,676 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:25:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:25:07,677 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:25:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:07,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742349_1525 (size=995) 2024-11-08T02:25:08,086 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T02:25:08,086 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T02:25:08,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:25:08,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, REOPEN/MOVE}] 2024-11-08T02:25:08,091 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, REOPEN/MOVE 2024-11-08T02:25:08,091 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=7fa5796b2a900c52ba2e3fc96ec46b12, regionState=CLOSING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, REOPEN/MOVE because future has completed 2024-11-08T02:25:08,093 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:25:08,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:25:08,245 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,245 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T02:25:08,245 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 7fa5796b2a900c52ba2e3fc96ec46b12, disabling compactions & flushes 2024-11-08T02:25:08,245 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:08,245 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:08,245 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. after waiting 0 ms 2024-11-08T02:25:08,245 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:08,248 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T02:25:08,248 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:08,248 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 7fa5796b2a900c52ba2e3fc96ec46b12: Waiting for close lock at 1731032708245Running coprocessor pre-close hooks at 1731032708245Disabling compacts and flushes for region at 1731032708245Disabling writes for close at 1731032708245Writing region close event to WAL at 1731032708245Running coprocessor post-close hooks at 1731032708248 (+3 ms)Closed at 1731032708248 2024-11-08T02:25:08,248 WARN [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 7fa5796b2a900c52ba2e3fc96ec46b12 to self. 2024-11-08T02:25:08,250 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=7fa5796b2a900c52ba2e3fc96ec46b12, regionState=CLOSED 2024-11-08T02:25:08,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:25:08,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-08T02:25:08,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 in 159 msec 2024-11-08T02:25:08,254 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, REOPEN/MOVE; state=CLOSED, location=331c7316141f,34325,1731032548147; forceNewPlan=false, retain=true 2024-11-08T02:25:08,404 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=7fa5796b2a900c52ba2e3fc96ec46b12, regionState=OPENING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, REOPEN/MOVE because future has completed 2024-11-08T02:25:08,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:25:08,559 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:08,559 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:25:08,560 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,560 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:25:08,560 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,560 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,561 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,561 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fa5796b2a900c52ba2e3fc96ec46b12 columnFamilyName A 2024-11-08T02:25:08,562 DEBUG [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:08,563 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(327): Store=7fa5796b2a900c52ba2e3fc96ec46b12/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:25:08,563 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,563 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fa5796b2a900c52ba2e3fc96ec46b12 columnFamilyName B 2024-11-08T02:25:08,563 DEBUG [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:08,564 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(327): Store=7fa5796b2a900c52ba2e3fc96ec46b12/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:25:08,564 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,564 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7fa5796b2a900c52ba2e3fc96ec46b12 columnFamilyName C 2024-11-08T02:25:08,564 DEBUG [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:08,564 INFO [StoreOpener-7fa5796b2a900c52ba2e3fc96ec46b12-1 {}] regionserver.HStore(327): Store=7fa5796b2a900c52ba2e3fc96ec46b12/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:25:08,564 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,565 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,566 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,566 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,566 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,567 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:25:08,568 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,568 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 7fa5796b2a900c52ba2e3fc96ec46b12; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70224826, jitterRate=0.04643145203590393}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:25:08,568 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,569 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 7fa5796b2a900c52ba2e3fc96ec46b12: Running coprocessor pre-open hook at 1731032708560Writing region info on filesystem at 1731032708560Initializing all the Stores at 1731032708560Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032708560Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032708561 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731032708561Cleaning up temporary data from old regions at 1731032708566 (+5 ms)Running coprocessor post-open hooks at 1731032708568 (+2 ms)Region opened successfully at 1731032708569 (+1 ms) 2024-11-08T02:25:08,569 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., pid=149, masterSystemTime=1731032708557 2024-11-08T02:25:08,571 DEBUG [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:08,571 INFO [RS_OPEN_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:08,571 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=7fa5796b2a900c52ba2e3fc96ec46b12, regionState=OPEN, openSeqNum=5, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:25:08,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-08T02:25:08,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 in 167 msec 2024-11-08T02:25:08,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-08T02:25:08,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, REOPEN/MOVE in 484 msec 2024-11-08T02:25:08,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-08T02:25:08,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-08T02:25:08,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 901 msec 2024-11-08T02:25:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-08T02:25:08,580 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:08,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:25:08,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:25:08,581 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:25:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:25:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:25:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fcc6e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:25:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:25:08,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:08,583 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:25:08,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3170e4c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:25:08,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:25:08,584 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:25:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:25:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:25:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f1b5c86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:25:08,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:25:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:08,585 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:25:08,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29fd7f5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:25:08,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:25:08,586 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:25:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:25:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:25:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47297c4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:25:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:25:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:08,587 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:25:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3343f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:25:08,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:25:08,589 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:25:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:25:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:25:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c30385, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:25:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:25:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:08,590 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:25:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c8472c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,590 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:25:08,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:25:08,591 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:25:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:25:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:25:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@646e0cf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:25:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:25:08,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:08,592 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:25:08,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e6192, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:25:08,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:25:08,594 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:25:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:25:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:25:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d52a577, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:25:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:25:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:08,595 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:25:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bae1e59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:25:08,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:25:08,596 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:25:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:25:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:25:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a38153a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:25:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:25:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:08,598 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:25:08,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ff232e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:25:08,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:25:08,599 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:25:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:25:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:25:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb161ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:25:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:25:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:08,601 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:25:08,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792c4b48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:25:08,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:25:08,602 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:25:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:25:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:25:08,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2340ebe5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:25:08,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:25:08,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:08,603 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:25:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed721b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 331c7316141f,41605,-1 for getting cluster id 2024-11-08T02:25:08,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T02:25:08,605 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad3e781f-9d7b-4875-a9cb-20151d78056f' 2024-11-08T02:25:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T02:25:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad3e781f-9d7b-4875-a9cb-20151d78056f" 2024-11-08T02:25:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76dff2e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [331c7316141f,41605,-1] 2024-11-08T02:25:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T02:25:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:08,606 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T02:25:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767018ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:25:08,607 DEBUG [Thread-2361 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:25:08,607 DEBUG [Thread-2367 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:25:08,607 DEBUG [Thread-2363 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:25:08,607 DEBUG [Thread-2352 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:25:08,607 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:25:08,608 DEBUG [Thread-2356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:25:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:25:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:25:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:25:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:25:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:25:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-08T02:25:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:25:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:25:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:25:08,608 DEBUG [Thread-2359 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:25:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:25:08,609 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:25:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:25:08,609 DEBUG [Thread-2348 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:25:08,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:25:08,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:25:08,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:25:08,609 DEBUG [Thread-2365 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:25:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:25:08,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:25:08,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:25:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T02:25:08,610 DEBUG [Thread-2350 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:25:08,610 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:25:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:25:08,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:25:08,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:25:08,610 DEBUG [Thread-2354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T02:25:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:25:08,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:25:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:25:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:25:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:25:08,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:25:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:25:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:25:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:25:08,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:25:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:25:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:25:08,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:25:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:25:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=331c7316141f,34325,1731032548147, seqNum=-1] 2024-11-08T02:25:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:25:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:25:08,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:25:08,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:25:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:25:08,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:25:08,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5] 2024-11-08T02:25:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:08,620 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:08,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:25:08,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081cd3105e9e6e43f094a3af0383a1e0cb_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032708619/Put/seqid=0 2024-11-08T02:25:08,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742350_1526 (size=17034) 2024-11-08T02:25:08,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032768671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032768672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:08,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032768675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:08,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032768676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:08,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032768688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T02:25:08,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T02:25:08,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:08,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:08,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:08,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:08,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:08,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032768787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032768787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:08,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032768788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:08,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032768788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:08,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032768798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:08,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T02:25:08,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:08,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:08,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:08,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:08,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T02:25:08,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032768997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:08,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032768998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:08,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032768998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032768998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032769008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,050 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:09,053 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081cd3105e9e6e43f094a3af0383a1e0cb_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081cd3105e9e6e43f094a3af0383a1e0cb_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:09,054 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/474e3a76e42e4a1c944f7c4c66bb437c, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:09,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/474e3a76e42e4a1c944f7c4c66bb437c is 175, key is test_row_0/A:col10/1731032708619/Put/seqid=0 2024-11-08T02:25:09,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742351_1527 (size=48139) 2024-11-08T02:25:09,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T02:25:09,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:09,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T02:25:09,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:09,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T02:25:09,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032769307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032769308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032769308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032769318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032769318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T02:25:09,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,457 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/474e3a76e42e4a1c944f7c4c66bb437c 2024-11-08T02:25:09,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/4a421d0dedbc4364bd956d6f1ad48842 is 50, key is test_row_0/B:col10/1731032708619/Put/seqid=0 2024-11-08T02:25:09,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742352_1528 (size=12001) 2024-11-08T02:25:09,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/4a421d0dedbc4364bd956d6f1ad48842 2024-11-08T02:25:09,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/37ec5c578b8541cc96a1cfc82d2b8119 is 50, key is test_row_0/C:col10/1731032708619/Put/seqid=0 2024-11-08T02:25:09,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742353_1529 (size=12001) 2024-11-08T02:25:09,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T02:25:09,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:09,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T02:25:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T02:25:09,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032769818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032769818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032769818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032769827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032769828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:09,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T02:25:09,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:09,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:09,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/37ec5c578b8541cc96a1cfc82d2b8119 2024-11-08T02:25:09,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/474e3a76e42e4a1c944f7c4c66bb437c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/474e3a76e42e4a1c944f7c4c66bb437c 2024-11-08T02:25:09,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/474e3a76e42e4a1c944f7c4c66bb437c, entries=250, sequenceid=18, filesize=47.0 K 2024-11-08T02:25:09,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/4a421d0dedbc4364bd956d6f1ad48842 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4a421d0dedbc4364bd956d6f1ad48842 2024-11-08T02:25:09,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4a421d0dedbc4364bd956d6f1ad48842, entries=150, sequenceid=18, filesize=11.7 K 2024-11-08T02:25:09,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/37ec5c578b8541cc96a1cfc82d2b8119 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/37ec5c578b8541cc96a1cfc82d2b8119 2024-11-08T02:25:09,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/37ec5c578b8541cc96a1cfc82d2b8119, entries=150, sequenceid=18, filesize=11.7 K 2024-11-08T02:25:09,916 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-08T02:25:09,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 1296ms, sequenceid=18, compaction requested=false 2024-11-08T02:25:09,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:09,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T02:25:09,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:09,985 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T02:25:09,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108dc0e96dbaafb480fb83156996903e435_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032708675/Put/seqid=0 2024-11-08T02:25:09,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742354_1530 (size=12154) 2024-11-08T02:25:09,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:09,997 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108dc0e96dbaafb480fb83156996903e435_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108dc0e96dbaafb480fb83156996903e435_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/831d7cb1f9c6451b9648a38781b3d1fc, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/831d7cb1f9c6451b9648a38781b3d1fc is 175, key is test_row_0/A:col10/1731032708675/Put/seqid=0 2024-11-08T02:25:10,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742355_1531 (size=30955) 2024-11-08T02:25:10,002 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/831d7cb1f9c6451b9648a38781b3d1fc 2024-11-08T02:25:10,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/56f9d326ea7f44cba49e8599eb08d0e4 is 50, key is test_row_0/B:col10/1731032708675/Put/seqid=0 2024-11-08T02:25:10,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742356_1532 (size=12001) 2024-11-08T02:25:10,412 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/56f9d326ea7f44cba49e8599eb08d0e4 2024-11-08T02:25:10,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/a375570f45b54e8ebf603454b7285e3d is 50, key is test_row_0/C:col10/1731032708675/Put/seqid=0 2024-11-08T02:25:10,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742357_1533 (size=12001) 2024-11-08T02:25:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T02:25:10,823 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/a375570f45b54e8ebf603454b7285e3d 2024-11-08T02:25:10,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/831d7cb1f9c6451b9648a38781b3d1fc as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/831d7cb1f9c6451b9648a38781b3d1fc 2024-11-08T02:25:10,829 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/831d7cb1f9c6451b9648a38781b3d1fc, entries=150, sequenceid=40, filesize=30.2 K 2024-11-08T02:25:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:10,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:10,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/56f9d326ea7f44cba49e8599eb08d0e4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/56f9d326ea7f44cba49e8599eb08d0e4 2024-11-08T02:25:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,833 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/56f9d326ea7f44cba49e8599eb08d0e4, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:25:10,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/a375570f45b54e8ebf603454b7285e3d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/a375570f45b54e8ebf603454b7285e3d 2024-11-08T02:25:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,840 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/a375570f45b54e8ebf603454b7285e3d, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:25:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,841 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=46.96 KB/48090 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 856ms, sequenceid=40, compaction requested=false 2024-11-08T02:25:10,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:10,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-08T02:25:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:10,842 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-08T02:25:10,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:25:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-08T02:25:10,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2320 sec 2024-11-08T02:25:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.2380 sec 2024-11-08T02:25:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fdd9f750beb34a1e8e13ef818f8a6859_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032710839/Put/seqid=0 2024-11-08T02:25:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742359_1535 (size=24358) 2024-11-08T02:25:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:10,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032770869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:10,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032770873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:10,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032770874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:10,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:10,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032770874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032770875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:10,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032770977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:10,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032770988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:10,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032770988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032770988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:10,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032770988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:11,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032771187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:11,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032771198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032771198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032771198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032771198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:11,218 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:25:11,251 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:11,255 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fdd9f750beb34a1e8e13ef818f8a6859_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fdd9f750beb34a1e8e13ef818f8a6859_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:11,256 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/2799698c910f40f99ec0ab41508bc80e, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:11,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/2799698c910f40f99ec0ab41508bc80e is 175, key is test_row_0/A:col10/1731032710839/Put/seqid=0 2024-11-08T02:25:11,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742358_1534 (size=73995) 2024-11-08T02:25:11,261 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/2799698c910f40f99ec0ab41508bc80e 2024-11-08T02:25:11,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/c152612366d44f84aa6d6b452307afb2 is 50, key is test_row_0/B:col10/1731032710839/Put/seqid=0 2024-11-08T02:25:11,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742360_1536 (size=12001) 2024-11-08T02:25:11,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032771498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032771507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032771507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:11,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032771508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:11,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032771508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:11,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/c152612366d44f84aa6d6b452307afb2 2024-11-08T02:25:11,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/7c06384e8d9b4b78928a68ceb638d93a is 50, key is test_row_0/C:col10/1731032710839/Put/seqid=0 2024-11-08T02:25:11,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742361_1537 (size=12001) 2024-11-08T02:25:11,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/7c06384e8d9b4b78928a68ceb638d93a 2024-11-08T02:25:11,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/2799698c910f40f99ec0ab41508bc80e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2799698c910f40f99ec0ab41508bc80e 2024-11-08T02:25:11,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2799698c910f40f99ec0ab41508bc80e, entries=400, sequenceid=51, filesize=72.3 K 2024-11-08T02:25:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/c152612366d44f84aa6d6b452307afb2 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c152612366d44f84aa6d6b452307afb2 2024-11-08T02:25:11,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c152612366d44f84aa6d6b452307afb2, entries=150, sequenceid=51, filesize=11.7 K 2024-11-08T02:25:11,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/7c06384e8d9b4b78928a68ceb638d93a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/7c06384e8d9b4b78928a68ceb638d93a 2024-11-08T02:25:11,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/7c06384e8d9b4b78928a68ceb638d93a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-08T02:25:11,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 858ms, sequenceid=51, compaction requested=true 2024-11-08T02:25:11,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:11,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:25:11,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:11,700 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:11,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:25:11,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:11,701 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:11,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:11,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:11,704 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:11,704 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 153089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:11,704 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/A is initiating minor compaction (all files) 2024-11-08T02:25:11,704 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/B is initiating minor compaction (all files) 2024-11-08T02:25:11,704 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/B in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:11,704 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/A in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:11,704 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4a421d0dedbc4364bd956d6f1ad48842, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/56f9d326ea7f44cba49e8599eb08d0e4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c152612366d44f84aa6d6b452307afb2] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=35.2 K 2024-11-08T02:25:11,704 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/474e3a76e42e4a1c944f7c4c66bb437c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/831d7cb1f9c6451b9648a38781b3d1fc, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2799698c910f40f99ec0ab41508bc80e] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=149.5 K 2024-11-08T02:25:11,704 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:11,704 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/474e3a76e42e4a1c944f7c4c66bb437c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/831d7cb1f9c6451b9648a38781b3d1fc, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2799698c910f40f99ec0ab41508bc80e] 2024-11-08T02:25:11,705 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a421d0dedbc4364bd956d6f1ad48842, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731032708618 2024-11-08T02:25:11,705 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 474e3a76e42e4a1c944f7c4c66bb437c, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731032708615 2024-11-08T02:25:11,705 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 56f9d326ea7f44cba49e8599eb08d0e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731032708644 2024-11-08T02:25:11,705 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 831d7cb1f9c6451b9648a38781b3d1fc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731032708644 2024-11-08T02:25:11,705 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c152612366d44f84aa6d6b452307afb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731032710835 2024-11-08T02:25:11,705 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2799698c910f40f99ec0ab41508bc80e, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731032710829 2024-11-08T02:25:11,715 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:11,716 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/fee1fd58922d49bd938b5f07d06b1763 is 50, key is test_row_0/B:col10/1731032710839/Put/seqid=0 2024-11-08T02:25:11,729 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:11,731 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108f1909e7f2d3b4f5cb059315175d9d9b9_7fa5796b2a900c52ba2e3fc96ec46b12 store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:11,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742362_1538 (size=12104) 2024-11-08T02:25:11,734 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108f1909e7f2d3b4f5cb059315175d9d9b9_7fa5796b2a900c52ba2e3fc96ec46b12, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:11,734 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f1909e7f2d3b4f5cb059315175d9d9b9_7fa5796b2a900c52ba2e3fc96ec46b12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:11,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742363_1539 (size=4469) 2024-11-08T02:25:11,747 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#A#compaction#460 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:11,748 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/a2dcacf290ea419d81334a8b2bbe56c8 is 175, key is test_row_0/A:col10/1731032710839/Put/seqid=0 2024-11-08T02:25:11,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742364_1540 (size=31058) 2024-11-08T02:25:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:12,017 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:12,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:25:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032772022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032772022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032772023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b55b473e419e42e7961db307c0ac693a_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032710864/Put/seqid=0 2024-11-08T02:25:12,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032772027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032772027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742365_1541 (size=14594) 2024-11-08T02:25:12,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,041 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b55b473e419e42e7961db307c0ac693a_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b55b473e419e42e7961db307c0ac693a_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:12,042 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/d9bdce40c7f14f4fa3c3546b23c57182, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:12,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/d9bdce40c7f14f4fa3c3546b23c57182 is 175, key is test_row_0/A:col10/1731032710864/Put/seqid=0 2024-11-08T02:25:12,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742366_1542 (size=39549) 2024-11-08T02:25:12,060 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/d9bdce40c7f14f4fa3c3546b23c57182 2024-11-08T02:25:12,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/25094e2e483b4d49b60a6837d8c2b49c is 50, key is test_row_0/B:col10/1731032710864/Put/seqid=0 2024-11-08T02:25:12,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742367_1543 (size=12001) 2024-11-08T02:25:12,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/25094e2e483b4d49b60a6837d8c2b49c 2024-11-08T02:25:12,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/71977075b17349559a60267bad08ab6f is 50, key is test_row_0/C:col10/1731032710864/Put/seqid=0 2024-11-08T02:25:12,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742368_1544 (size=12001) 2024-11-08T02:25:12,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/71977075b17349559a60267bad08ab6f 2024-11-08T02:25:12,140 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/fee1fd58922d49bd938b5f07d06b1763 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/fee1fd58922d49bd938b5f07d06b1763 2024-11-08T02:25:12,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/d9bdce40c7f14f4fa3c3546b23c57182 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d9bdce40c7f14f4fa3c3546b23c57182 2024-11-08T02:25:12,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032772140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032772141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032772141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032772141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032772141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,146 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/B of 7fa5796b2a900c52ba2e3fc96ec46b12 into fee1fd58922d49bd938b5f07d06b1763(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:12,146 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:12,146 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/B, priority=13, startTime=1731032711700; duration=0sec 2024-11-08T02:25:12,146 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:12,146 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:B 2024-11-08T02:25:12,146 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:12,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d9bdce40c7f14f4fa3c3546b23c57182, entries=200, sequenceid=77, filesize=38.6 K 2024-11-08T02:25:12,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/25094e2e483b4d49b60a6837d8c2b49c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/25094e2e483b4d49b60a6837d8c2b49c 2024-11-08T02:25:12,147 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:12,147 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/C is initiating minor compaction (all files) 2024-11-08T02:25:12,147 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/C in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:12,147 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/37ec5c578b8541cc96a1cfc82d2b8119, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/a375570f45b54e8ebf603454b7285e3d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/7c06384e8d9b4b78928a68ceb638d93a] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=35.2 K 2024-11-08T02:25:12,148 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ec5c578b8541cc96a1cfc82d2b8119, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731032708618 2024-11-08T02:25:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,155 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a375570f45b54e8ebf603454b7285e3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731032708644 2024-11-08T02:25:12,155 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c06384e8d9b4b78928a68ceb638d93a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731032710835 2024-11-08T02:25:12,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/25094e2e483b4d49b60a6837d8c2b49c, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T02:25:12,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/71977075b17349559a60267bad08ab6f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/71977075b17349559a60267bad08ab6f 2024-11-08T02:25:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,162 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#C#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/71977075b17349559a60267bad08ab6f, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T02:25:12,163 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/b8738b49190a4a309bea85e8a9379425 is 50, key is test_row_0/C:col10/1731032710839/Put/seqid=0 2024-11-08T02:25:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 150ms, sequenceid=77, compaction requested=false 2024-11-08T02:25:12,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742369_1545 (size=12104) 2024-11-08T02:25:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,173 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/b8738b49190a4a309bea85e8a9379425 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/b8738b49190a4a309bea85e8a9379425 2024-11-08T02:25:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,177 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/C of 7fa5796b2a900c52ba2e3fc96ec46b12 into b8738b49190a4a309bea85e8a9379425(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:12,177 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:12,177 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/C, priority=13, startTime=1731032711701; duration=0sec 2024-11-08T02:25:12,177 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:12,177 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:C 2024-11-08T02:25:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,178 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/a2dcacf290ea419d81334a8b2bbe56c8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a2dcacf290ea419d81334a8b2bbe56c8 2024-11-08T02:25:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,184 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/A of 7fa5796b2a900c52ba2e3fc96ec46b12 into a2dcacf290ea419d81334a8b2bbe56c8(size=30.3 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:12,184 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:12,184 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/A, priority=13, startTime=1731032711700; duration=0sec 2024-11-08T02:25:12,184 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:12,184 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:A 2024-11-08T02:25:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,353 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:12,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:25:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108003018248d8d4b5ca919722acff84fb9_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_1/A:col10/1731032712351/Put/seqid=0 2024-11-08T02:25:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742370_1546 (size=12154) 2024-11-08T02:25:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032772421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032772425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032772425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032772427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032772427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032772538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032772538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032772538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032772538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032772538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032772748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032772748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032772748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032772748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032772748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T02:25:12,768 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:12,768 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:25:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-08T02:25:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T02:25:12,769 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:25:12,770 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:25:12,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:25:12,771 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:12,787 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108003018248d8d4b5ca919722acff84fb9_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108003018248d8d4b5ca919722acff84fb9_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:12,788 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/9413832df9d4474cb0e864b3b74262af, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:12,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/9413832df9d4474cb0e864b3b74262af is 175, key is test_row_1/A:col10/1731032712351/Put/seqid=0 2024-11-08T02:25:12,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742371_1547 (size=30951) 2024-11-08T02:25:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T02:25:12,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T02:25:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:12,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:12,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:13,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032773058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032773058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032773058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032773058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032773058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T02:25:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:13,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:13,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T02:25:13,193 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/9413832df9d4474cb0e864b3b74262af 2024-11-08T02:25:13,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/7469dcc32da54b0b9f487d5d4df1a0bd is 50, key is test_row_1/B:col10/1731032712351/Put/seqid=0 2024-11-08T02:25:13,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742372_1548 (size=9657) 2024-11-08T02:25:13,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/7469dcc32da54b0b9f487d5d4df1a0bd 2024-11-08T02:25:13,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/1a2e204c87d54bddbc59c1f7fcfa3928 is 50, key is test_row_1/C:col10/1731032712351/Put/seqid=0 2024-11-08T02:25:13,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742373_1549 (size=9657) 2024-11-08T02:25:13,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/1a2e204c87d54bddbc59c1f7fcfa3928 2024-11-08T02:25:13,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/9413832df9d4474cb0e864b3b74262af as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9413832df9d4474cb0e864b3b74262af 2024-11-08T02:25:13,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9413832df9d4474cb0e864b3b74262af, entries=150, sequenceid=91, filesize=30.2 K 2024-11-08T02:25:13,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/7469dcc32da54b0b9f487d5d4df1a0bd as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7469dcc32da54b0b9f487d5d4df1a0bd 2024-11-08T02:25:13,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7469dcc32da54b0b9f487d5d4df1a0bd, entries=100, sequenceid=91, filesize=9.4 K 2024-11-08T02:25:13,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/1a2e204c87d54bddbc59c1f7fcfa3928 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/1a2e204c87d54bddbc59c1f7fcfa3928 2024-11-08T02:25:13,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/1a2e204c87d54bddbc59c1f7fcfa3928, entries=100, sequenceid=91, filesize=9.4 K 2024-11-08T02:25:13,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 868ms, sequenceid=91, compaction requested=true 2024-11-08T02:25:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:25:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:13,221 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:25:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:13,221 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:13,222 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101558 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:13,222 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:13,222 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/A is initiating minor compaction (all files) 2024-11-08T02:25:13,222 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/B is initiating minor compaction (all files) 2024-11-08T02:25:13,222 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/B in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:13,222 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/A in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:13,222 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/fee1fd58922d49bd938b5f07d06b1763, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/25094e2e483b4d49b60a6837d8c2b49c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7469dcc32da54b0b9f487d5d4df1a0bd] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=33.0 K 2024-11-08T02:25:13,222 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a2dcacf290ea419d81334a8b2bbe56c8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d9bdce40c7f14f4fa3c3546b23c57182, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9413832df9d4474cb0e864b3b74262af] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=99.2 K 2024-11-08T02:25:13,222 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:13,222 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a2dcacf290ea419d81334a8b2bbe56c8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d9bdce40c7f14f4fa3c3546b23c57182, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9413832df9d4474cb0e864b3b74262af] 2024-11-08T02:25:13,222 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting fee1fd58922d49bd938b5f07d06b1763, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731032710835 2024-11-08T02:25:13,222 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2dcacf290ea419d81334a8b2bbe56c8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731032710835 2024-11-08T02:25:13,223 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 25094e2e483b4d49b60a6837d8c2b49c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731032710864 2024-11-08T02:25:13,224 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9bdce40c7f14f4fa3c3546b23c57182, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731032710864 2024-11-08T02:25:13,224 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 7469dcc32da54b0b9f487d5d4df1a0bd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731032712026 2024-11-08T02:25:13,224 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9413832df9d4474cb0e864b3b74262af, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731032712025 2024-11-08T02:25:13,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T02:25:13,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:13,228 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:25:13,229 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:13,231 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#B#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:13,231 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/d94bf9fbe92e43a7b09a52eb42706c21 is 50, key is test_row_0/B:col10/1731032710864/Put/seqid=0 2024-11-08T02:25:13,234 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108e309147dfea140e7bca5180769018ab0_7fa5796b2a900c52ba2e3fc96ec46b12 store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:13,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742374_1550 (size=12207) 2024-11-08T02:25:13,236 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108e309147dfea140e7bca5180769018ab0_7fa5796b2a900c52ba2e3fc96ec46b12, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:13,236 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e309147dfea140e7bca5180769018ab0_7fa5796b2a900c52ba2e3fc96ec46b12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:13,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ec41cc27b22641c1a11a30657f9582fe_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032712425/Put/seqid=0 2024-11-08T02:25:13,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742375_1551 (size=4469) 2024-11-08T02:25:13,242 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/d94bf9fbe92e43a7b09a52eb42706c21 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/d94bf9fbe92e43a7b09a52eb42706c21 2024-11-08T02:25:13,242 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#A#compaction#468 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:13,243 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/a0e3c3b12ecc49cfb283d392a005348e is 175, key is test_row_0/A:col10/1731032710864/Put/seqid=0 2024-11-08T02:25:13,247 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/B of 7fa5796b2a900c52ba2e3fc96ec46b12 into d94bf9fbe92e43a7b09a52eb42706c21(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:13,248 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:13,248 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/B, priority=13, startTime=1731032713221; duration=0sec 2024-11-08T02:25:13,248 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:13,248 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:B 2024-11-08T02:25:13,248 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:13,249 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:13,249 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/C is initiating minor compaction (all files) 2024-11-08T02:25:13,249 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/C in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:13,250 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/b8738b49190a4a309bea85e8a9379425, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/71977075b17349559a60267bad08ab6f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/1a2e204c87d54bddbc59c1f7fcfa3928] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=33.0 K 2024-11-08T02:25:13,250 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting b8738b49190a4a309bea85e8a9379425, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731032710835 2024-11-08T02:25:13,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742376_1552 (size=12154) 2024-11-08T02:25:13,251 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 71977075b17349559a60267bad08ab6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731032710864 2024-11-08T02:25:13,251 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a2e204c87d54bddbc59c1f7fcfa3928, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731032712026 2024-11-08T02:25:13,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:13,255 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ec41cc27b22641c1a11a30657f9582fe_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ec41cc27b22641c1a11a30657f9582fe_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:13,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/20aa8ccdb3f04f5485a0f602e2c85d8d, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:13,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/20aa8ccdb3f04f5485a0f602e2c85d8d is 175, key is test_row_0/A:col10/1731032712425/Put/seqid=0 2024-11-08T02:25:13,266 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:13,267 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/e315664a3a734f9ab5591e233b031fd5 is 50, key is test_row_0/C:col10/1731032710864/Put/seqid=0 2024-11-08T02:25:13,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742377_1553 (size=31268) 2024-11-08T02:25:13,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742378_1554 (size=30955) 2024-11-08T02:25:13,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742379_1555 (size=12207) 2024-11-08T02:25:13,304 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/e315664a3a734f9ab5591e233b031fd5 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/e315664a3a734f9ab5591e233b031fd5 2024-11-08T02:25:13,312 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/C of 7fa5796b2a900c52ba2e3fc96ec46b12 into e315664a3a734f9ab5591e233b031fd5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:13,313 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:13,313 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/C, priority=13, startTime=1731032713221; duration=0sec 2024-11-08T02:25:13,313 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:13,313 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:C 2024-11-08T02:25:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T02:25:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:13,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:13,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032773573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032773573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032773573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032773575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032773576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,687 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/20aa8ccdb3f04f5485a0f602e2c85d8d 2024-11-08T02:25:13,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032773688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032773688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032773688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032773688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,692 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/a0e3c3b12ecc49cfb283d392a005348e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a0e3c3b12ecc49cfb283d392a005348e 2024-11-08T02:25:13,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032773691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a10be4f4de81486da1e23f2caf3a777f is 50, key is test_row_0/B:col10/1731032712425/Put/seqid=0 2024-11-08T02:25:13,698 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/A of 7fa5796b2a900c52ba2e3fc96ec46b12 into a0e3c3b12ecc49cfb283d392a005348e(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:13,698 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:13,698 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/A, priority=13, startTime=1731032713221; duration=0sec 2024-11-08T02:25:13,698 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:13,698 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:A 2024-11-08T02:25:13,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742380_1556 (size=12001) 2024-11-08T02:25:13,700 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a10be4f4de81486da1e23f2caf3a777f 2024-11-08T02:25:13,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/cb4ed301a82b46c38542d0bbea5f8e9b is 50, key is test_row_0/C:col10/1731032712425/Put/seqid=0 2024-11-08T02:25:13,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742381_1557 (size=12001) 2024-11-08T02:25:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032773898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032773898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032773898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032773898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032773898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:13,902 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T02:25:14,125 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/cb4ed301a82b46c38542d0bbea5f8e9b 2024-11-08T02:25:14,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/20aa8ccdb3f04f5485a0f602e2c85d8d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/20aa8ccdb3f04f5485a0f602e2c85d8d 2024-11-08T02:25:14,132 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/20aa8ccdb3f04f5485a0f602e2c85d8d, entries=150, sequenceid=117, filesize=30.2 K 2024-11-08T02:25:14,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a10be4f4de81486da1e23f2caf3a777f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a10be4f4de81486da1e23f2caf3a777f 2024-11-08T02:25:14,135 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a10be4f4de81486da1e23f2caf3a777f, entries=150, sequenceid=117, filesize=11.7 K 2024-11-08T02:25:14,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/cb4ed301a82b46c38542d0bbea5f8e9b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/cb4ed301a82b46c38542d0bbea5f8e9b 2024-11-08T02:25:14,139 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/cb4ed301a82b46c38542d0bbea5f8e9b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-08T02:25:14,140 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 911ms, sequenceid=117, compaction requested=false 2024-11-08T02:25:14,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:14,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:14,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-08T02:25:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-08T02:25:14,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-08T02:25:14,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3710 sec 2024-11-08T02:25:14,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.3750 sec 2024-11-08T02:25:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:14,210 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:14,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:25:14,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ca5775f9bef54d6899144ac5a0e8b24b_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032714208/Put/seqid=0 2024-11-08T02:25:14,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742382_1558 (size=12254) 2024-11-08T02:25:14,241 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:14,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032774239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,245 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ca5775f9bef54d6899144ac5a0e8b24b_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ca5775f9bef54d6899144ac5a0e8b24b_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:14,246 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/13c9b8a5f5194e5ebbce5e0de5264a42, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:14,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/13c9b8a5f5194e5ebbce5e0de5264a42 is 175, key is test_row_0/A:col10/1731032714208/Put/seqid=0 2024-11-08T02:25:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032774240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032774242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032774243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032774244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742383_1559 (size=31055) 2024-11-08T02:25:14,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032774348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032774358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032774358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032774358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032774359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032774558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032774568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032774568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032774568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032774568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,655 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/13c9b8a5f5194e5ebbce5e0de5264a42 2024-11-08T02:25:14,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/997ffbad2c084417bca2aaf77b8a18a6 is 50, key is test_row_0/B:col10/1731032714208/Put/seqid=0 2024-11-08T02:25:14,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742384_1560 (size=12101) 2024-11-08T02:25:14,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032774868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032774878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032774878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032774878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032774878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:14,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T02:25:14,918 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:14,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:25:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-08T02:25:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T02:25:14,920 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:25:14,920 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:25:14,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:25:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T02:25:15,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/997ffbad2c084417bca2aaf77b8a18a6 2024-11-08T02:25:15,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/0736298ff9a349ffabd10e4e673cf9e7 is 50, key is test_row_0/C:col10/1731032714208/Put/seqid=0 2024-11-08T02:25:15,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742385_1561 (size=12101) 2024-11-08T02:25:15,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-08T02:25:15,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:15,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:15,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:15,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:15,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:15,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-08T02:25:15,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:15,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:15,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:15,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:15,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T02:25:15,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-08T02:25:15,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:15,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:15,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:15,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:15,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:15,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032775378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:15,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:15,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:15,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:15,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032775388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:15,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032775388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:15,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032775388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:15,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032775388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:15,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/0736298ff9a349ffabd10e4e673cf9e7 2024-11-08T02:25:15,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/13c9b8a5f5194e5ebbce5e0de5264a42 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/13c9b8a5f5194e5ebbce5e0de5264a42 2024-11-08T02:25:15,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/13c9b8a5f5194e5ebbce5e0de5264a42, entries=150, sequenceid=133, filesize=30.3 K 2024-11-08T02:25:15,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/997ffbad2c084417bca2aaf77b8a18a6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/997ffbad2c084417bca2aaf77b8a18a6 2024-11-08T02:25:15,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/997ffbad2c084417bca2aaf77b8a18a6, entries=150, sequenceid=133, filesize=11.8 K 2024-11-08T02:25:15,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/0736298ff9a349ffabd10e4e673cf9e7 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0736298ff9a349ffabd10e4e673cf9e7 2024-11-08T02:25:15,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0736298ff9a349ffabd10e4e673cf9e7, entries=150, sequenceid=133, filesize=11.8 K 2024-11-08T02:25:15,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 1276ms, sequenceid=133, compaction requested=true 2024-11-08T02:25:15,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:15,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:25:15,486 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:15,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:15,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:25:15,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:15,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:15,487 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:15,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:15,488 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:15,488 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/B is initiating minor compaction (all files) 2024-11-08T02:25:15,488 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/B in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:15,488 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93278 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:15,488 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/A is initiating minor compaction (all files) 2024-11-08T02:25:15,488 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/A in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:15,488 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a0e3c3b12ecc49cfb283d392a005348e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/20aa8ccdb3f04f5485a0f602e2c85d8d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/13c9b8a5f5194e5ebbce5e0de5264a42] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=91.1 K 2024-11-08T02:25:15,488 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/d94bf9fbe92e43a7b09a52eb42706c21, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a10be4f4de81486da1e23f2caf3a777f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/997ffbad2c084417bca2aaf77b8a18a6] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=35.5 K 2024-11-08T02:25:15,488 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:15,489 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a0e3c3b12ecc49cfb283d392a005348e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/20aa8ccdb3f04f5485a0f602e2c85d8d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/13c9b8a5f5194e5ebbce5e0de5264a42] 2024-11-08T02:25:15,489 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting d94bf9fbe92e43a7b09a52eb42706c21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731032710864 2024-11-08T02:25:15,489 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0e3c3b12ecc49cfb283d392a005348e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731032710864 2024-11-08T02:25:15,489 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a10be4f4de81486da1e23f2caf3a777f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731032712423 2024-11-08T02:25:15,489 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20aa8ccdb3f04f5485a0f602e2c85d8d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731032712423 2024-11-08T02:25:15,489 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13c9b8a5f5194e5ebbce5e0de5264a42, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731032713575 2024-11-08T02:25:15,489 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 997ffbad2c084417bca2aaf77b8a18a6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731032713575 2024-11-08T02:25:15,495 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:15,496 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:15,496 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/56fcc5438c6845598a2810fa66246209 is 50, key is test_row_0/B:col10/1731032714208/Put/seqid=0 2024-11-08T02:25:15,496 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108aa49ee901bee4a0fb359e68758765a3b_7fa5796b2a900c52ba2e3fc96ec46b12 store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:15,498 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108aa49ee901bee4a0fb359e68758765a3b_7fa5796b2a900c52ba2e3fc96ec46b12, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:15,498 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108aa49ee901bee4a0fb359e68758765a3b_7fa5796b2a900c52ba2e3fc96ec46b12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:15,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742386_1562 (size=12409) 2024-11-08T02:25:15,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742387_1563 (size=4469) 2024-11-08T02:25:15,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-08T02:25:15,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:15,530 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T02:25:15,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cee40935b24a472bb304a9847f9b96cc_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032714239/Put/seqid=0 2024-11-08T02:25:15,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742388_1564 (size=12304) 2024-11-08T02:25:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T02:25:15,903 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:15,904 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/2b9fc1ac28a34d2c9f7f3794bc049fba is 175, key is test_row_0/A:col10/1731032714208/Put/seqid=0 2024-11-08T02:25:15,905 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/56fcc5438c6845598a2810fa66246209 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/56fcc5438c6845598a2810fa66246209 2024-11-08T02:25:15,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742389_1565 (size=31363) 2024-11-08T02:25:15,910 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/B of 7fa5796b2a900c52ba2e3fc96ec46b12 into 56fcc5438c6845598a2810fa66246209(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:15,910 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:15,910 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/B, priority=13, startTime=1731032715487; duration=0sec 2024-11-08T02:25:15,910 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:15,910 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:B 2024-11-08T02:25:15,910 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:15,924 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/2b9fc1ac28a34d2c9f7f3794bc049fba as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2b9fc1ac28a34d2c9f7f3794bc049fba 2024-11-08T02:25:15,924 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:15,925 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/C is initiating minor compaction (all files) 2024-11-08T02:25:15,925 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/C in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:15,925 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/e315664a3a734f9ab5591e233b031fd5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/cb4ed301a82b46c38542d0bbea5f8e9b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0736298ff9a349ffabd10e4e673cf9e7] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=35.5 K 2024-11-08T02:25:15,926 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting e315664a3a734f9ab5591e233b031fd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731032710864 2024-11-08T02:25:15,926 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting cb4ed301a82b46c38542d0bbea5f8e9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731032712423 2024-11-08T02:25:15,927 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 0736298ff9a349ffabd10e4e673cf9e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731032713575 2024-11-08T02:25:15,931 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/A of 7fa5796b2a900c52ba2e3fc96ec46b12 into 2b9fc1ac28a34d2c9f7f3794bc049fba(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:15,931 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:15,931 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/A, priority=13, startTime=1731032715486; duration=0sec 2024-11-08T02:25:15,931 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:15,931 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:A 2024-11-08T02:25:15,937 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#C#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:15,938 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/8201f65f702140ccbe4560df9c20bffa is 50, key is test_row_0/C:col10/1731032714208/Put/seqid=0 2024-11-08T02:25:15,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:15,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742390_1566 (size=12409) 2024-11-08T02:25:15,943 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cee40935b24a472bb304a9847f9b96cc_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cee40935b24a472bb304a9847f9b96cc_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f5ac7426fcf04146aa893d0d6ffdbdd0, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:15,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f5ac7426fcf04146aa893d0d6ffdbdd0 is 175, key is test_row_0/A:col10/1731032714239/Put/seqid=0 2024-11-08T02:25:15,946 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/8201f65f702140ccbe4560df9c20bffa as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/8201f65f702140ccbe4560df9c20bffa 2024-11-08T02:25:15,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742391_1567 (size=31105) 2024-11-08T02:25:15,951 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/C of 7fa5796b2a900c52ba2e3fc96ec46b12 into 8201f65f702140ccbe4560df9c20bffa(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:15,951 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:15,951 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/C, priority=13, startTime=1731032715487; duration=0sec 2024-11-08T02:25:15,951 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:15,951 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:C 2024-11-08T02:25:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T02:25:16,347 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f5ac7426fcf04146aa893d0d6ffdbdd0 2024-11-08T02:25:16,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/846caddb279145caa09c2537dba845a7 is 50, key is test_row_0/B:col10/1731032714239/Put/seqid=0 2024-11-08T02:25:16,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742392_1568 (size=12151) 2024-11-08T02:25:16,366 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/846caddb279145caa09c2537dba845a7 2024-11-08T02:25:16,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/8f5401a1c6b04c939421a2c8fc5c0b7e is 50, key is test_row_0/C:col10/1731032714239/Put/seqid=0 2024-11-08T02:25:16,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742393_1569 (size=12151) 2024-11-08T02:25:16,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:16,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032776402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032776402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032776432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032776432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032776433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032776538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032776538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032776538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032776548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032776548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032776748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032776748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032776748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032776757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032776758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:16,778 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/8f5401a1c6b04c939421a2c8fc5c0b7e 2024-11-08T02:25:16,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f5ac7426fcf04146aa893d0d6ffdbdd0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f5ac7426fcf04146aa893d0d6ffdbdd0 2024-11-08T02:25:16,785 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f5ac7426fcf04146aa893d0d6ffdbdd0, entries=150, sequenceid=156, filesize=30.4 K 2024-11-08T02:25:16,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/846caddb279145caa09c2537dba845a7 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/846caddb279145caa09c2537dba845a7 2024-11-08T02:25:16,788 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/846caddb279145caa09c2537dba845a7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-08T02:25:16,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/8f5401a1c6b04c939421a2c8fc5c0b7e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/8f5401a1c6b04c939421a2c8fc5c0b7e 2024-11-08T02:25:16,792 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/8f5401a1c6b04c939421a2c8fc5c0b7e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-08T02:25:16,792 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 1262ms, sequenceid=156, compaction requested=false 2024-11-08T02:25:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-08T02:25:16,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-08T02:25:16,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-08T02:25:16,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8740 sec 2024-11-08T02:25:16,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8780 sec 2024-11-08T02:25:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:17,060 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:17,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T02:25:17,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108125eafd5330748cd93efd338582e8589_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032716407/Put/seqid=0 2024-11-08T02:25:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T02:25:17,068 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:17,068 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:25:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-08T02:25:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T02:25:17,070 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:25:17,070 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:25:17,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:25:17,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742394_1570 (size=14794) 2024-11-08T02:25:17,174 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T02:25:17,179 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108125eafd5330748cd93efd338582e8589_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108125eafd5330748cd93efd338582e8589_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:17,180 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/9d4571fb48c544beb2c8628666d789bd, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:17,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/9d4571fb48c544beb2c8628666d789bd is 175, key is test_row_0/A:col10/1731032716407/Put/seqid=0 2024-11-08T02:25:17,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032777174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032777174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032777174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032777176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032777176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742395_1571 (size=39749) 2024-11-08T02:25:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/9d4571fb48c544beb2c8628666d789bd 2024-11-08T02:25:17,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/f195db62591b4bbd94832bfe6434be33 is 50, key is test_row_0/B:col10/1731032716407/Put/seqid=0 2024-11-08T02:25:17,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742396_1572 (size=12151) 2024-11-08T02:25:17,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T02:25:17,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:17,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032777288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032777288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032777288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032777288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032777288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T02:25:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T02:25:17,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032777497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032777497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032777497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032777498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032777498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T02:25:17,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/f195db62591b4bbd94832bfe6434be33 2024-11-08T02:25:17,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/fd532c38a06c4b3d9d48c1d8a6428887 is 50, key is test_row_0/C:col10/1731032716407/Put/seqid=0 2024-11-08T02:25:17,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742397_1573 (size=12151) 2024-11-08T02:25:17,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T02:25:17,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:17,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T02:25:17,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032777808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032777808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032777808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032777808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032777809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:17,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T02:25:17,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:17,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T02:25:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:17,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:18,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/fd532c38a06c4b3d9d48c1d8a6428887 2024-11-08T02:25:18,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/9d4571fb48c544beb2c8628666d789bd as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9d4571fb48c544beb2c8628666d789bd 2024-11-08T02:25:18,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9d4571fb48c544beb2c8628666d789bd, entries=200, sequenceid=175, filesize=38.8 K 2024-11-08T02:25:18,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/f195db62591b4bbd94832bfe6434be33 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f195db62591b4bbd94832bfe6434be33 2024-11-08T02:25:18,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f195db62591b4bbd94832bfe6434be33, entries=150, sequenceid=175, filesize=11.9 K 2024-11-08T02:25:18,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/fd532c38a06c4b3d9d48c1d8a6428887 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/fd532c38a06c4b3d9d48c1d8a6428887 2024-11-08T02:25:18,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/fd532c38a06c4b3d9d48c1d8a6428887, entries=150, sequenceid=175, filesize=11.9 K 2024-11-08T02:25:18,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 963ms, sequenceid=175, compaction requested=true 2024-11-08T02:25:18,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:18,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:25:18,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:18,023 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:18,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:25:18,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:18,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:18,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T02:25:18,023 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:18,024 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:18,024 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/A is initiating minor compaction (all files) 2024-11-08T02:25:18,024 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:18,024 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/A in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:18,024 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/B is initiating minor compaction (all files) 2024-11-08T02:25:18,024 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2b9fc1ac28a34d2c9f7f3794bc049fba, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f5ac7426fcf04146aa893d0d6ffdbdd0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9d4571fb48c544beb2c8628666d789bd] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=99.8 K 2024-11-08T02:25:18,024 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/B in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:18,024 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:18,024 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2b9fc1ac28a34d2c9f7f3794bc049fba, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f5ac7426fcf04146aa893d0d6ffdbdd0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9d4571fb48c544beb2c8628666d789bd] 2024-11-08T02:25:18,024 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/56fcc5438c6845598a2810fa66246209, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/846caddb279145caa09c2537dba845a7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f195db62591b4bbd94832bfe6434be33] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=35.9 K 2024-11-08T02:25:18,025 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b9fc1ac28a34d2c9f7f3794bc049fba, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731032713575 2024-11-08T02:25:18,025 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5ac7426fcf04146aa893d0d6ffdbdd0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731032714239 2024-11-08T02:25:18,025 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 56fcc5438c6845598a2810fa66246209, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731032713575 2024-11-08T02:25:18,025 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d4571fb48c544beb2c8628666d789bd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731032716402 2024-11-08T02:25:18,025 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 846caddb279145caa09c2537dba845a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731032714239 2024-11-08T02:25:18,026 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting f195db62591b4bbd94832bfe6434be33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731032716402 2024-11-08T02:25:18,030 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:18,031 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:18,032 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/0ee1e38f043b46fdbebbb6b810194b83 is 50, key is test_row_0/B:col10/1731032716407/Put/seqid=0 2024-11-08T02:25:18,034 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411085fc54f45fcf54b5ca9f023fadbf1ef47_7fa5796b2a900c52ba2e3fc96ec46b12 store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:18,036 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411085fc54f45fcf54b5ca9f023fadbf1ef47_7fa5796b2a900c52ba2e3fc96ec46b12, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:18,036 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085fc54f45fcf54b5ca9f023fadbf1ef47_7fa5796b2a900c52ba2e3fc96ec46b12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:18,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742398_1574 (size=12561) 2024-11-08T02:25:18,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742399_1575 (size=4469) 2024-11-08T02:25:18,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T02:25:18,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:18,137 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:25:18,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085e86c066ddee47d5aed6283b8c81b24e_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032717077/Put/seqid=0 2024-11-08T02:25:18,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742400_1576 (size=12304) 2024-11-08T02:25:18,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,152 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085e86c066ddee47d5aed6283b8c81b24e_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085e86c066ddee47d5aed6283b8c81b24e_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:18,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/d81247b96dbd43f9a81b9fdc2c031516, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:18,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/d81247b96dbd43f9a81b9fdc2c031516 is 175, key is test_row_0/A:col10/1731032717077/Put/seqid=0 2024-11-08T02:25:18,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742401_1577 (size=31105) 2024-11-08T02:25:18,159 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/d81247b96dbd43f9a81b9fdc2c031516 2024-11-08T02:25:18,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/d0a807625c58410d9b1ed265c8755aa4 is 50, key is test_row_0/B:col10/1731032717077/Put/seqid=0 2024-11-08T02:25:18,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742402_1578 (size=12151) 2024-11-08T02:25:18,174 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/d0a807625c58410d9b1ed265c8755aa4 2024-11-08T02:25:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/0050c60c737f439497f071be265533f2 is 50, key is test_row_0/C:col10/1731032717077/Put/seqid=0 2024-11-08T02:25:18,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742403_1579 (size=12151) 2024-11-08T02:25:18,194 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/0050c60c737f439497f071be265533f2 2024-11-08T02:25:18,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/d81247b96dbd43f9a81b9fdc2c031516 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d81247b96dbd43f9a81b9fdc2c031516 2024-11-08T02:25:18,202 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d81247b96dbd43f9a81b9fdc2c031516, entries=150, sequenceid=195, filesize=30.4 K 2024-11-08T02:25:18,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/d0a807625c58410d9b1ed265c8755aa4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/d0a807625c58410d9b1ed265c8755aa4 2024-11-08T02:25:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,207 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/d0a807625c58410d9b1ed265c8755aa4, entries=150, sequenceid=195, filesize=11.9 K 2024-11-08T02:25:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/0050c60c737f439497f071be265533f2 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0050c60c737f439497f071be265533f2 2024-11-08T02:25:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T02:25:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,211 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0050c60c737f439497f071be265533f2, entries=150, sequenceid=195, filesize=11.9 K 2024-11-08T02:25:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,212 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 75ms, sequenceid=195, compaction requested=true 2024-11-08T02:25:18,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:18,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-08T02:25:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-08T02:25:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-08T02:25:18,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1430 sec 2024-11-08T02:25:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.1480 sec 2024-11-08T02:25:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:18,355 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:18,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:25:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088156d53ed6524161851e1fa33ebb35ac_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032718355/Put/seqid=0 2024-11-08T02:25:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742405_1581 (size=27248) 2024-11-08T02:25:18,373 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,376 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088156d53ed6524161851e1fa33ebb35ac_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088156d53ed6524161851e1fa33ebb35ac_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:18,376 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/1a818718f1e2473d9b4eb06aa722a602, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:18,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/1a818718f1e2473d9b4eb06aa722a602 is 175, key is test_row_0/A:col10/1731032718355/Put/seqid=0 2024-11-08T02:25:18,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742404_1580 (size=83034) 2024-11-08T02:25:18,379 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/1a818718f1e2473d9b4eb06aa722a602 2024-11-08T02:25:18,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/3ddb4292b36142609a4050733c0b3fff is 50, key is test_row_0/B:col10/1731032718355/Put/seqid=0 2024-11-08T02:25:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742406_1582 (size=12151) 2024-11-08T02:25:18,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/3ddb4292b36142609a4050733c0b3fff 2024-11-08T02:25:18,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/0aa1a289ef4041b0a66423fa65dfefed is 50, key is test_row_0/C:col10/1731032718355/Put/seqid=0 2024-11-08T02:25:18,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032778389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032778390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032778393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742407_1583 (size=12151) 2024-11-08T02:25:18,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/0aa1a289ef4041b0a66423fa65dfefed 2024-11-08T02:25:18,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032778393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032778396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/1a818718f1e2473d9b4eb06aa722a602 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1a818718f1e2473d9b4eb06aa722a602 2024-11-08T02:25:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1a818718f1e2473d9b4eb06aa722a602, entries=450, sequenceid=207, filesize=81.1 K 2024-11-08T02:25:18,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/3ddb4292b36142609a4050733c0b3fff as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/3ddb4292b36142609a4050733c0b3fff 2024-11-08T02:25:18,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/3ddb4292b36142609a4050733c0b3fff, entries=150, sequenceid=207, filesize=11.9 K 2024-11-08T02:25:18,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/0aa1a289ef4041b0a66423fa65dfefed as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0aa1a289ef4041b0a66423fa65dfefed 2024-11-08T02:25:18,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0aa1a289ef4041b0a66423fa65dfefed, entries=150, sequenceid=207, filesize=11.9 K 2024-11-08T02:25:18,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 62ms, sequenceid=207, compaction requested=true 2024-11-08T02:25:18,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:18,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:A, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:18,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T02:25:18,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:B, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:18,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-08T02:25:18,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:18,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-08T02:25:18,443 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/0ee1e38f043b46fdbebbb6b810194b83 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/0ee1e38f043b46fdbebbb6b810194b83 2024-11-08T02:25:18,443 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:18,444 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/b0cabf49df3b40bdab49c1b60cf11ef3 is 175, key is test_row_0/A:col10/1731032716407/Put/seqid=0 2024-11-08T02:25:18,447 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/B of 7fa5796b2a900c52ba2e3fc96ec46b12 into 0ee1e38f043b46fdbebbb6b810194b83(size=12.3 K), total size for store is 36.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:18,448 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:18,448 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/B, priority=13, startTime=1731032718023; duration=0sec 2024-11-08T02:25:18,448 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-08T02:25:18,448 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:B 2024-11-08T02:25:18,448 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-08T02:25:18,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742408_1584 (size=31515) 2024-11-08T02:25:18,449 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T02:25:18,449 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T02:25:18,449 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. because compaction request was cancelled 2024-11-08T02:25:18,449 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:A 2024-11-08T02:25:18,449 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T02:25:18,451 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61013 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T02:25:18,451 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/C is initiating minor compaction (all files) 2024-11-08T02:25:18,451 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/C in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:18,451 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/8201f65f702140ccbe4560df9c20bffa, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/8f5401a1c6b04c939421a2c8fc5c0b7e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/fd532c38a06c4b3d9d48c1d8a6428887, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0050c60c737f439497f071be265533f2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0aa1a289ef4041b0a66423fa65dfefed] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=59.6 K 2024-11-08T02:25:18,451 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 8201f65f702140ccbe4560df9c20bffa, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731032713575 2024-11-08T02:25:18,452 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5401a1c6b04c939421a2c8fc5c0b7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731032714239 2024-11-08T02:25:18,452 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting fd532c38a06c4b3d9d48c1d8a6428887, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731032716402 2024-11-08T02:25:18,453 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 0050c60c737f439497f071be265533f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032717077 2024-11-08T02:25:18,453 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aa1a289ef4041b0a66423fa65dfefed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731032718354 2024-11-08T02:25:18,468 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#C#compaction#494 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:18,469 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/c2d1b5bfba3b4e61887a22c204ff3475 is 50, key is test_row_0/C:col10/1731032718355/Put/seqid=0 2024-11-08T02:25:18,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742409_1585 (size=12629) 2024-11-08T02:25:18,479 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/c2d1b5bfba3b4e61887a22c204ff3475 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/c2d1b5bfba3b4e61887a22c204ff3475 2024-11-08T02:25:18,486 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/C of 7fa5796b2a900c52ba2e3fc96ec46b12 into c2d1b5bfba3b4e61887a22c204ff3475(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:18,486 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:18,486 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/C, priority=11, startTime=1731032718417; duration=0sec 2024-11-08T02:25:18,486 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T02:25:18,486 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:C 2024-11-08T02:25:18,486 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:B 2024-11-08T02:25:18,486 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:C 2024-11-08T02:25:18,486 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:18,492 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:18,492 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/B is initiating minor compaction (all files) 2024-11-08T02:25:18,493 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/B in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:18,493 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/0ee1e38f043b46fdbebbb6b810194b83, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/d0a807625c58410d9b1ed265c8755aa4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/3ddb4292b36142609a4050733c0b3fff] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=36.0 K 2024-11-08T02:25:18,493 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ee1e38f043b46fdbebbb6b810194b83, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731032716402 2024-11-08T02:25:18,493 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a807625c58410d9b1ed265c8755aa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032717077 2024-11-08T02:25:18,493 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ddb4292b36142609a4050733c0b3fff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731032718354 2024-11-08T02:25:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:18,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:25:18,507 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#B#compaction#495 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:18,508 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/7ad462d075174f4d96ec76f7c00cad90 is 50, key is test_row_0/B:col10/1731032718355/Put/seqid=0 2024-11-08T02:25:18,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084b9baefe478946049fdb614dc3e806ef_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032718500/Put/seqid=0 2024-11-08T02:25:18,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742410_1586 (size=12663) 2024-11-08T02:25:18,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742411_1587 (size=14794) 2024-11-08T02:25:18,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032778514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032778514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032778515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032778515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032778516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032778628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032778628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032778628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032778628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032778628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032778838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032778838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032778838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032778838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032778838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:18,852 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/b0cabf49df3b40bdab49c1b60cf11ef3 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b0cabf49df3b40bdab49c1b60cf11ef3 2024-11-08T02:25:18,855 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/A of 7fa5796b2a900c52ba2e3fc96ec46b12 into b0cabf49df3b40bdab49c1b60cf11ef3(size=30.8 K), total size for store is 142.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:18,855 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:18,855 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/A, priority=13, startTime=1731032718023; duration=0sec 2024-11-08T02:25:18,855 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:18,855 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:A 2024-11-08T02:25:18,855 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-08T02:25:18,855 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T02:25:18,855 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T02:25:18,855 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. because compaction request was cancelled 2024-11-08T02:25:18,855 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:C 2024-11-08T02:25:18,918 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:18,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084b9baefe478946049fdb614dc3e806ef_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084b9baefe478946049fdb614dc3e806ef_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:18,922 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/7ad462d075174f4d96ec76f7c00cad90 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7ad462d075174f4d96ec76f7c00cad90 2024-11-08T02:25:18,922 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/3aa81feb7fff43ada21f880b6d699b90, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:18,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/3aa81feb7fff43ada21f880b6d699b90 is 175, key is test_row_0/A:col10/1731032718500/Put/seqid=0 2024-11-08T02:25:18,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742412_1588 (size=39749) 2024-11-08T02:25:18,926 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/B of 7fa5796b2a900c52ba2e3fc96ec46b12 into 7ad462d075174f4d96ec76f7c00cad90(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:18,926 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:18,926 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/B, priority=13, startTime=1731032718417; duration=0sec 2024-11-08T02:25:18,926 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:18,926 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:B 2024-11-08T02:25:18,927 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/3aa81feb7fff43ada21f880b6d699b90 2024-11-08T02:25:18,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/4e24fc096aa5490eb97a711630c60f1a is 50, key is test_row_0/B:col10/1731032718500/Put/seqid=0 2024-11-08T02:25:18,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742413_1589 (size=12151) 2024-11-08T02:25:18,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/4e24fc096aa5490eb97a711630c60f1a 2024-11-08T02:25:18,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/44a1a00f0d484481a0f0c3b58f3fb086 is 50, key is test_row_0/C:col10/1731032718500/Put/seqid=0 2024-11-08T02:25:18,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742414_1590 (size=12151) 2024-11-08T02:25:19,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032779148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032779148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032779148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032779148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032779148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T02:25:19,218 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:19,218 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:25:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-08T02:25:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T02:25:19,220 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:25:19,220 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:25:19,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:25:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T02:25:19,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/44a1a00f0d484481a0f0c3b58f3fb086 2024-11-08T02:25:19,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/3aa81feb7fff43ada21f880b6d699b90 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3aa81feb7fff43ada21f880b6d699b90 2024-11-08T02:25:19,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3aa81feb7fff43ada21f880b6d699b90, entries=200, sequenceid=235, filesize=38.8 K 2024-11-08T02:25:19,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/4e24fc096aa5490eb97a711630c60f1a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4e24fc096aa5490eb97a711630c60f1a 2024-11-08T02:25:19,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4e24fc096aa5490eb97a711630c60f1a, entries=150, sequenceid=235, filesize=11.9 K 2024-11-08T02:25:19,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/44a1a00f0d484481a0f0c3b58f3fb086 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/44a1a00f0d484481a0f0c3b58f3fb086 2024-11-08T02:25:19,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/44a1a00f0d484481a0f0c3b58f3fb086, entries=150, sequenceid=235, filesize=11.9 K 2024-11-08T02:25:19,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 856ms, sequenceid=235, compaction requested=true 2024-11-08T02:25:19,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:19,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:25:19,357 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:25:19,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:19,357 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-08T02:25:19,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:25:19,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:19,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:19,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. because compaction request was cancelled 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:B 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 185403 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/A is initiating minor compaction (all files) 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T02:25:19,358 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/A in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. because compaction request was cancelled 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:C 2024-11-08T02:25:19,358 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b0cabf49df3b40bdab49c1b60cf11ef3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d81247b96dbd43f9a81b9fdc2c031516, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1a818718f1e2473d9b4eb06aa722a602, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3aa81feb7fff43ada21f880b6d699b90] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=181.1 K 2024-11-08T02:25:19,358 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b0cabf49df3b40bdab49c1b60cf11ef3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d81247b96dbd43f9a81b9fdc2c031516, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1a818718f1e2473d9b4eb06aa722a602, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3aa81feb7fff43ada21f880b6d699b90] 2024-11-08T02:25:19,358 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0cabf49df3b40bdab49c1b60cf11ef3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731032716402 2024-11-08T02:25:19,359 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting d81247b96dbd43f9a81b9fdc2c031516, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731032717077 2024-11-08T02:25:19,359 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a818718f1e2473d9b4eb06aa722a602, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731032718341 2024-11-08T02:25:19,359 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa81feb7fff43ada21f880b6d699b90, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032718382 2024-11-08T02:25:19,365 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:19,368 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108d71153bd8f6b451a82e405034e98e720_7fa5796b2a900c52ba2e3fc96ec46b12 store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:19,371 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108d71153bd8f6b451a82e405034e98e720_7fa5796b2a900c52ba2e3fc96ec46b12, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:19,371 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d71153bd8f6b451a82e405034e98e720_7fa5796b2a900c52ba2e3fc96ec46b12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:19,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-08T02:25:19,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:19,372 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T02:25:19,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742415_1591 (size=4469) 2024-11-08T02:25:19,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108baf0a4f3528a4ebbafa374e670709610_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032718514/Put/seqid=0 2024-11-08T02:25:19,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742416_1592 (size=12304) 2024-11-08T02:25:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T02:25:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:19,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:19,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032779715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032779715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032779716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032779717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032779717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,777 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#A#compaction#499 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:19,777 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/3dcbaa40447f477592ddc419b48e3656 is 175, key is test_row_0/A:col10/1731032718500/Put/seqid=0 2024-11-08T02:25:19,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742417_1593 (size=31651) 2024-11-08T02:25:19,783 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/3dcbaa40447f477592ddc419b48e3656 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3dcbaa40447f477592ddc419b48e3656 2024-11-08T02:25:19,787 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/A of 7fa5796b2a900c52ba2e3fc96ec46b12 into 3dcbaa40447f477592ddc419b48e3656(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:19,787 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:19,787 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/A, priority=12, startTime=1731032719357; duration=0sec 2024-11-08T02:25:19,787 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:19,787 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:A 2024-11-08T02:25:19,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:19,810 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108baf0a4f3528a4ebbafa374e670709610_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108baf0a4f3528a4ebbafa374e670709610_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:19,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/151fecc1c75a4866bc01e1dda69e5cc3, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:19,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/151fecc1c75a4866bc01e1dda69e5cc3 is 175, key is test_row_0/A:col10/1731032718514/Put/seqid=0 2024-11-08T02:25:19,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742418_1594 (size=31105) 2024-11-08T02:25:19,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032779827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032779828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032779828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032779828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032779828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T02:25:20,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032780037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032780038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032780038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032780038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032780038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,214 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/151fecc1c75a4866bc01e1dda69e5cc3 2024-11-08T02:25:20,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/705f211f404546a88bd7bcc91bd8aed6 is 50, key is test_row_0/B:col10/1731032718514/Put/seqid=0 2024-11-08T02:25:20,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742419_1595 (size=12151) 2024-11-08T02:25:20,222 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/705f211f404546a88bd7bcc91bd8aed6 2024-11-08T02:25:20,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/854ce21612b94752a3a71e66db330162 is 50, key is test_row_0/C:col10/1731032718514/Put/seqid=0 2024-11-08T02:25:20,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742420_1596 (size=12151) 2024-11-08T02:25:20,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032780348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032780348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032780348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032780348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032780348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T02:25:20,631 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/854ce21612b94752a3a71e66db330162 2024-11-08T02:25:20,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/151fecc1c75a4866bc01e1dda69e5cc3 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/151fecc1c75a4866bc01e1dda69e5cc3 2024-11-08T02:25:20,637 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/151fecc1c75a4866bc01e1dda69e5cc3, entries=150, sequenceid=247, filesize=30.4 K 2024-11-08T02:25:20,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/705f211f404546a88bd7bcc91bd8aed6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/705f211f404546a88bd7bcc91bd8aed6 2024-11-08T02:25:20,641 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/705f211f404546a88bd7bcc91bd8aed6, entries=150, sequenceid=247, filesize=11.9 K 2024-11-08T02:25:20,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/854ce21612b94752a3a71e66db330162 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/854ce21612b94752a3a71e66db330162 2024-11-08T02:25:20,644 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/854ce21612b94752a3a71e66db330162, entries=150, sequenceid=247, filesize=11.9 K 2024-11-08T02:25:20,645 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 1272ms, sequenceid=247, compaction requested=true 2024-11-08T02:25:20,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:20,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:20,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-08T02:25:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-08T02:25:20,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-08T02:25:20,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-11-08T02:25:20,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4290 sec 2024-11-08T02:25:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:20,862 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:20,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T02:25:20,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032780863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032780863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032780864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032780864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032780869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a90e3d677e634a3f96445f0a71f311ab_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032719715/Put/seqid=0 2024-11-08T02:25:20,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742421_1597 (size=12454) 2024-11-08T02:25:20,895 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:20,900 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a90e3d677e634a3f96445f0a71f311ab_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a90e3d677e634a3f96445f0a71f311ab_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:20,901 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/1d70b0e75802453f8b71ea495051af99, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:20,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/1d70b0e75802453f8b71ea495051af99 is 175, key is test_row_0/A:col10/1731032719715/Put/seqid=0 2024-11-08T02:25:20,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742422_1598 (size=31255) 2024-11-08T02:25:20,907 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/1d70b0e75802453f8b71ea495051af99 2024-11-08T02:25:20,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a386760b162544d9b9cc57afe3b91269 is 50, key is test_row_0/B:col10/1731032719715/Put/seqid=0 2024-11-08T02:25:20,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742423_1599 (size=12301) 2024-11-08T02:25:20,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032780978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032780978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032780978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032780978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032780978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:21,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032781188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032781189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:21,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032781189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:21,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032781189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:21,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032781189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:21,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a386760b162544d9b9cc57afe3b91269 2024-11-08T02:25:21,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/364c937e69bf4cb1953838bf65aecf97 is 50, key is test_row_0/C:col10/1731032719715/Put/seqid=0 2024-11-08T02:25:21,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742424_1600 (size=12301) 2024-11-08T02:25:21,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T02:25:21,368 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:21,368 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:25:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-08T02:25:21,371 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:25:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T02:25:21,372 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:25:21,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:25:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T02:25:21,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032781498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:21,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032781499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:21,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032781499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032781499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:21,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032781500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:21,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T02:25:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:21,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:21,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T02:25:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:21,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T02:25:21,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/364c937e69bf4cb1953838bf65aecf97 2024-11-08T02:25:21,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/1d70b0e75802453f8b71ea495051af99 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1d70b0e75802453f8b71ea495051af99 2024-11-08T02:25:21,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1d70b0e75802453f8b71ea495051af99, entries=150, sequenceid=278, filesize=30.5 K 2024-11-08T02:25:21,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a386760b162544d9b9cc57afe3b91269 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a386760b162544d9b9cc57afe3b91269 2024-11-08T02:25:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a386760b162544d9b9cc57afe3b91269, entries=150, sequenceid=278, filesize=12.0 K 2024-11-08T02:25:21,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/364c937e69bf4cb1953838bf65aecf97 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/364c937e69bf4cb1953838bf65aecf97 2024-11-08T02:25:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/364c937e69bf4cb1953838bf65aecf97, entries=150, sequenceid=278, filesize=12.0 K 2024-11-08T02:25:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 903ms, sequenceid=278, compaction requested=true 2024-11-08T02:25:21,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,765 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:21,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:25:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,766 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:25:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,766 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:21,766 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/A is initiating minor compaction (all files) 2024-11-08T02:25:21,767 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/A in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,767 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3dcbaa40447f477592ddc419b48e3656, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/151fecc1c75a4866bc01e1dda69e5cc3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1d70b0e75802453f8b71ea495051af99] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=91.8 K 2024-11-08T02:25:21,767 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:21,767 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3dcbaa40447f477592ddc419b48e3656, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/151fecc1c75a4866bc01e1dda69e5cc3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1d70b0e75802453f8b71ea495051af99] 2024-11-08T02:25:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,767 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dcbaa40447f477592ddc419b48e3656, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032718382 2024-11-08T02:25:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,768 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 151fecc1c75a4866bc01e1dda69e5cc3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731032718513 2024-11-08T02:25:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,768 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:25:21,768 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/B is initiating minor compaction (all files) 2024-11-08T02:25:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,768 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/B in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:21,768 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7ad462d075174f4d96ec76f7c00cad90, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4e24fc096aa5490eb97a711630c60f1a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/705f211f404546a88bd7bcc91bd8aed6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a386760b162544d9b9cc57afe3b91269] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=48.1 K 2024-11-08T02:25:21,768 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d70b0e75802453f8b71ea495051af99, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731032719715 2024-11-08T02:25:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:25:21,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:21,769 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ad462d075174f4d96ec76f7c00cad90, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731032718354 2024-11-08T02:25:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,769 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e24fc096aa5490eb97a711630c60f1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032718382 2024-11-08T02:25:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,770 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 705f211f404546a88bd7bcc91bd8aed6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731032718513 2024-11-08T02:25:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,770 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a386760b162544d9b9cc57afe3b91269, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731032719715 2024-11-08T02:25:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,777 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,786 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#B#compaction#507 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,787 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/01e18519853d455997384c934f6826d4 is 50, key is test_row_0/B:col10/1731032719715/Put/seqid=0 2024-11-08T02:25:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,788 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108a6ccb26b6cb245cc8f1e86e990c53165_7fa5796b2a900c52ba2e3fc96ec46b12 store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,790 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108a6ccb26b6cb245cc8f1e86e990c53165_7fa5796b2a900c52ba2e3fc96ec46b12, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,790 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a6ccb26b6cb245cc8f1e86e990c53165_7fa5796b2a900c52ba2e3fc96ec46b12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742425_1601 (size=12949) 2024-11-08T02:25:21,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742426_1602 (size=4469) 2024-11-08T02:25:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,820 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#A#compaction#506 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,820 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/982143b5de4a4a328f5d865763a72d32 is 175, key is test_row_0/A:col10/1731032719715/Put/seqid=0 2024-11-08T02:25:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742427_1603 (size=31903) 2024-11-08T02:25:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T02:25:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,838 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T02:25:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110827c7a157f013442b9fed784fa7ecf8bb_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032720867/Put/seqid=0 2024-11-08T02:25:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742428_1604 (size=9914) 2024-11-08T02:25:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,871 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110827c7a157f013442b9fed784fa7ecf8bb_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110827c7a157f013442b9fed784fa7ecf8bb_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/eb07ef3e3e854264a3166e0080ee8c00, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/eb07ef3e3e854264a3166e0080ee8c00 is 175, key is test_row_0/A:col10/1731032720867/Put/seqid=0 2024-11-08T02:25:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742429_1605 (size=22561) 2024-11-08T02:25:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,911 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/eb07ef3e3e854264a3166e0080ee8c00 2024-11-08T02:25:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a363ce5a6654420385783ae71c5f0a78 is 50, key is test_row_0/B:col10/1731032720867/Put/seqid=0 2024-11-08T02:25:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742430_1606 (size=9857) 2024-11-08T02:25:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T02:25:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032782095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032782095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032782097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032782100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032782102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032782219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032782219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032782219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,224 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/01e18519853d455997384c934f6826d4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/01e18519853d455997384c934f6826d4 2024-11-08T02:25:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032782219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032782219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,230 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/B of 7fa5796b2a900c52ba2e3fc96ec46b12 into 01e18519853d455997384c934f6826d4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:22,230 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:22,230 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/B, priority=12, startTime=1731032721766; duration=0sec 2024-11-08T02:25:22,230 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:22,230 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:B 2024-11-08T02:25:22,230 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:25:22,231 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/982143b5de4a4a328f5d865763a72d32 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/982143b5de4a4a328f5d865763a72d32 2024-11-08T02:25:22,231 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:25:22,231 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/C is initiating minor compaction (all files) 2024-11-08T02:25:22,231 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/C in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:22,231 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/c2d1b5bfba3b4e61887a22c204ff3475, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/44a1a00f0d484481a0f0c3b58f3fb086, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/854ce21612b94752a3a71e66db330162, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/364c937e69bf4cb1953838bf65aecf97] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=48.1 K 2024-11-08T02:25:22,232 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c2d1b5bfba3b4e61887a22c204ff3475, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731032718354 2024-11-08T02:25:22,232 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 44a1a00f0d484481a0f0c3b58f3fb086, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731032718382 2024-11-08T02:25:22,232 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 854ce21612b94752a3a71e66db330162, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731032718513 2024-11-08T02:25:22,233 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 364c937e69bf4cb1953838bf65aecf97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731032719715 2024-11-08T02:25:22,235 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/A of 7fa5796b2a900c52ba2e3fc96ec46b12 into 982143b5de4a4a328f5d865763a72d32(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:22,235 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:22,235 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/A, priority=13, startTime=1731032721765; duration=0sec 2024-11-08T02:25:22,235 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:22,235 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:A 2024-11-08T02:25:22,259 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#C#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:22,259 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/7d596f6924064f8eb30967904e24bbd7 is 50, key is test_row_0/C:col10/1731032719715/Put/seqid=0 2024-11-08T02:25:22,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742431_1607 (size=12915) 2024-11-08T02:25:22,305 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/7d596f6924064f8eb30967904e24bbd7 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/7d596f6924064f8eb30967904e24bbd7 2024-11-08T02:25:22,313 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/C of 7fa5796b2a900c52ba2e3fc96ec46b12 into 7d596f6924064f8eb30967904e24bbd7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:22,313 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:22,313 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/C, priority=12, startTime=1731032721769; duration=0sec 2024-11-08T02:25:22,313 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:22,313 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:C 2024-11-08T02:25:22,379 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a363ce5a6654420385783ae71c5f0a78 2024-11-08T02:25:22,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/2cfeddcac190453badd7d54d0a7d406c is 50, key is test_row_0/C:col10/1731032720867/Put/seqid=0 2024-11-08T02:25:22,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032782428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032782428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032782428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032782429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742432_1608 (size=9857) 2024-11-08T02:25:22,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032782438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,445 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/2cfeddcac190453badd7d54d0a7d406c 2024-11-08T02:25:22,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/eb07ef3e3e854264a3166e0080ee8c00 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eb07ef3e3e854264a3166e0080ee8c00 2024-11-08T02:25:22,452 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eb07ef3e3e854264a3166e0080ee8c00, entries=100, sequenceid=284, filesize=22.0 K 2024-11-08T02:25:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a363ce5a6654420385783ae71c5f0a78 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a363ce5a6654420385783ae71c5f0a78 2024-11-08T02:25:22,457 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a363ce5a6654420385783ae71c5f0a78, entries=100, sequenceid=284, filesize=9.6 K 2024-11-08T02:25:22,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/2cfeddcac190453badd7d54d0a7d406c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/2cfeddcac190453badd7d54d0a7d406c 2024-11-08T02:25:22,464 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/2cfeddcac190453badd7d54d0a7d406c, entries=100, sequenceid=284, filesize=9.6 K 2024-11-08T02:25:22,465 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 627ms, sequenceid=284, compaction requested=false 2024-11-08T02:25:22,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:22,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:22,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-08T02:25:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-08T02:25:22,470 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-08T02:25:22,470 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0950 sec 2024-11-08T02:25:22,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.1020 sec 2024-11-08T02:25:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T02:25:22,508 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:22,509 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:25:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-08T02:25:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T02:25:22,516 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:25:22,517 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:25:22,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:25:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T02:25:22,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-08T02:25:22,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:22,670 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-08T02:25:22,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110865a8f9d3c7494102978abbef2de6d1fd_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032722097/Put/seqid=0 2024-11-08T02:25:22,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742433_1609 (size=12454) 2024-11-08T02:25:22,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:22,727 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110865a8f9d3c7494102978abbef2de6d1fd_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110865a8f9d3c7494102978abbef2de6d1fd_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:22,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/48a96b2caa8541288ecd30c7d97be223, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:22,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/48a96b2caa8541288ecd30c7d97be223 is 175, key is test_row_0/A:col10/1731032722097/Put/seqid=0 2024-11-08T02:25:22,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742434_1610 (size=31255) 2024-11-08T02:25:22,737 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/48a96b2caa8541288ecd30c7d97be223 2024-11-08T02:25:22,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a22c3014f5654f4294313208d6b51dec is 50, key is test_row_0/B:col10/1731032722097/Put/seqid=0 2024-11-08T02:25:22,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032782745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032782747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032782747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032782747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032782747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742435_1611 (size=12301) 2024-11-08T02:25:22,770 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a22c3014f5654f4294313208d6b51dec 2024-11-08T02:25:22,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/6421633ed5624f5993fbe9aca0dee41a is 50, key is test_row_0/C:col10/1731032722097/Put/seqid=0 2024-11-08T02:25:22,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742436_1612 (size=12301) 2024-11-08T02:25:22,798 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/6421633ed5624f5993fbe9aca0dee41a 2024-11-08T02:25:22,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/48a96b2caa8541288ecd30c7d97be223 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/48a96b2caa8541288ecd30c7d97be223 2024-11-08T02:25:22,808 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/48a96b2caa8541288ecd30c7d97be223, entries=150, sequenceid=317, filesize=30.5 K 2024-11-08T02:25:22,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/a22c3014f5654f4294313208d6b51dec as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a22c3014f5654f4294313208d6b51dec 2024-11-08T02:25:22,818 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a22c3014f5654f4294313208d6b51dec, entries=150, sequenceid=317, filesize=12.0 K 2024-11-08T02:25:22,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/6421633ed5624f5993fbe9aca0dee41a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/6421633ed5624f5993fbe9aca0dee41a 2024-11-08T02:25:22,824 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/6421633ed5624f5993fbe9aca0dee41a, entries=150, sequenceid=317, filesize=12.0 K 2024-11-08T02:25:22,825 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 155ms, sequenceid=317, compaction requested=true 2024-11-08T02:25:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-08T02:25:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-08T02:25:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T02:25:22,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-08T02:25:22,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 309 msec 2024-11-08T02:25:22,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 322 msec 2024-11-08T02:25:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:22,861 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:22,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:25:22,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086e37403b63884f8090b7102a03bf1214_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032722860/Put/seqid=0 2024-11-08T02:25:22,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742437_1613 (size=17534) 2024-11-08T02:25:22,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032782896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032782898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032782901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:22,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032782902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:22,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,908 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032783008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032783008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032783008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032783018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T02:25:23,138 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:23,138 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:25:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-08T02:25:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T02:25:23,140 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:25:23,140 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:25:23,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:25:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032783217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032783217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032783218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032783228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T02:25:23,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032783257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:23,278 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086e37403b63884f8090b7102a03bf1214_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086e37403b63884f8090b7102a03bf1214_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:23,279 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/e27ead74d1194f14ab01044c30881c00, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:23,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/e27ead74d1194f14ab01044c30881c00 is 175, key is test_row_0/A:col10/1731032722860/Put/seqid=0 2024-11-08T02:25:23,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742438_1614 (size=48639) 2024-11-08T02:25:23,287 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/e27ead74d1194f14ab01044c30881c00 2024-11-08T02:25:23,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-08T02:25:23,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:23,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:23,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:23,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/c51e06d98cbc4a359d2b0b7cf17c7bb6 is 50, key is test_row_0/B:col10/1731032722860/Put/seqid=0 2024-11-08T02:25:23,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742439_1615 (size=12301) 2024-11-08T02:25:23,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-08T02:25:23,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:23,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:23,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:23,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T02:25:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032783527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032783528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032783528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032783538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:23,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-08T02:25:23,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:23,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:23,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:23,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/c51e06d98cbc4a359d2b0b7cf17c7bb6 2024-11-08T02:25:23,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/28638651d4f940d58028561daf83e361 is 50, key is test_row_0/C:col10/1731032722860/Put/seqid=0 2024-11-08T02:25:23,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742440_1616 (size=12301) 2024-11-08T02:25:23,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-08T02:25:23,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:23,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:23,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:23,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T02:25:23,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-08T02:25:23,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:23,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:23,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:23,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:23,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:24,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032784038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:24,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032784038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:24,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032784038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:24,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032784048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:24,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-08T02:25:24,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:24,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:24,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:24,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:24,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:25:24,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/28638651d4f940d58028561daf83e361 2024-11-08T02:25:24,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/e27ead74d1194f14ab01044c30881c00 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/e27ead74d1194f14ab01044c30881c00 2024-11-08T02:25:24,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/e27ead74d1194f14ab01044c30881c00, entries=250, sequenceid=328, filesize=47.5 K 2024-11-08T02:25:24,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/c51e06d98cbc4a359d2b0b7cf17c7bb6 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c51e06d98cbc4a359d2b0b7cf17c7bb6 2024-11-08T02:25:24,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c51e06d98cbc4a359d2b0b7cf17c7bb6, entries=150, sequenceid=328, filesize=12.0 K 2024-11-08T02:25:24,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/28638651d4f940d58028561daf83e361 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/28638651d4f940d58028561daf83e361 2024-11-08T02:25:24,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/28638651d4f940d58028561daf83e361, entries=150, sequenceid=328, filesize=12.0 K 2024-11-08T02:25:24,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 1283ms, sequenceid=328, compaction requested=true 2024-11-08T02:25:24,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:24,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:25:24,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:24,145 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:25:24,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:25:24,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:24,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:24,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:24,145 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:25:24,146 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134358 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:25:24,146 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:25:24,146 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/B is initiating minor compaction (all files) 2024-11-08T02:25:24,146 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/A is initiating minor compaction (all files) 2024-11-08T02:25:24,146 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/B in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:24,146 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/A in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:24,146 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/982143b5de4a4a328f5d865763a72d32, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eb07ef3e3e854264a3166e0080ee8c00, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/48a96b2caa8541288ecd30c7d97be223, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/e27ead74d1194f14ab01044c30881c00] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=131.2 K 2024-11-08T02:25:24,146 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/01e18519853d455997384c934f6826d4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a363ce5a6654420385783ae71c5f0a78, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a22c3014f5654f4294313208d6b51dec, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c51e06d98cbc4a359d2b0b7cf17c7bb6] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=46.3 K 2024-11-08T02:25:24,146 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:24,146 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/982143b5de4a4a328f5d865763a72d32, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eb07ef3e3e854264a3166e0080ee8c00, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/48a96b2caa8541288ecd30c7d97be223, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/e27ead74d1194f14ab01044c30881c00] 2024-11-08T02:25:24,147 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 982143b5de4a4a328f5d865763a72d32, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731032719715 2024-11-08T02:25:24,147 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e18519853d455997384c934f6826d4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731032719715 2024-11-08T02:25:24,147 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb07ef3e3e854264a3166e0080ee8c00, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731032720863 2024-11-08T02:25:24,147 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a363ce5a6654420385783ae71c5f0a78, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731032720863 2024-11-08T02:25:24,147 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a96b2caa8541288ecd30c7d97be223, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731032722091 2024-11-08T02:25:24,147 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting a22c3014f5654f4294313208d6b51dec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731032722091 2024-11-08T02:25:24,147 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting e27ead74d1194f14ab01044c30881c00, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731032722745 2024-11-08T02:25:24,147 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c51e06d98cbc4a359d2b0b7cf17c7bb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731032722860 2024-11-08T02:25:24,155 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:24,157 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#B#compaction#519 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:24,158 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/7ca212d7eb0e4229901edeeb58eabf75 is 50, key is test_row_0/B:col10/1731032722860/Put/seqid=0 2024-11-08T02:25:24,160 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108219399bceee94023b105487736508ea1_7fa5796b2a900c52ba2e3fc96ec46b12 store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:24,161 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108219399bceee94023b105487736508ea1_7fa5796b2a900c52ba2e3fc96ec46b12, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:24,162 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108219399bceee94023b105487736508ea1_7fa5796b2a900c52ba2e3fc96ec46b12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:24,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-08T02:25:24,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:24,208 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:25:24,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742441_1617 (size=13085) 2024-11-08T02:25:24,216 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/7ca212d7eb0e4229901edeeb58eabf75 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7ca212d7eb0e4229901edeeb58eabf75 2024-11-08T02:25:24,220 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/B of 7fa5796b2a900c52ba2e3fc96ec46b12 into 7ca212d7eb0e4229901edeeb58eabf75(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:24,220 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:24,220 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/B, priority=12, startTime=1731032724145; duration=0sec 2024-11-08T02:25:24,220 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:24,220 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:B 2024-11-08T02:25:24,220 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:25:24,221 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:25:24,221 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/C is initiating minor compaction (all files) 2024-11-08T02:25:24,221 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/C in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:24,221 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/7d596f6924064f8eb30967904e24bbd7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/2cfeddcac190453badd7d54d0a7d406c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/6421633ed5624f5993fbe9aca0dee41a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/28638651d4f940d58028561daf83e361] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=46.3 K 2024-11-08T02:25:24,222 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d596f6924064f8eb30967904e24bbd7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731032719715 2024-11-08T02:25:24,223 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cfeddcac190453badd7d54d0a7d406c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731032720863 2024-11-08T02:25:24,224 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 6421633ed5624f5993fbe9aca0dee41a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731032722091 2024-11-08T02:25:24,224 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 28638651d4f940d58028561daf83e361, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731032722860 2024-11-08T02:25:24,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742442_1618 (size=4469) 2024-11-08T02:25:24,233 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#A#compaction#518 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:24,234 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/34681517e9494107a0984febfb615d92 is 175, key is test_row_0/A:col10/1731032722860/Put/seqid=0 2024-11-08T02:25:24,239 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#C#compaction#520 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:24,239 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/d74f13ca4b3b41a1a3cef8246a5bb1f5 is 50, key is test_row_0/C:col10/1731032722860/Put/seqid=0 2024-11-08T02:25:24,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108286a711e7fc14d0fb918eae79d6f9a65_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032722895/Put/seqid=0 2024-11-08T02:25:24,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742443_1619 (size=32039) 2024-11-08T02:25:24,258 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/34681517e9494107a0984febfb615d92 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/34681517e9494107a0984febfb615d92 2024-11-08T02:25:24,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742444_1620 (size=13051) 2024-11-08T02:25:24,262 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/A of 7fa5796b2a900c52ba2e3fc96ec46b12 into 34681517e9494107a0984febfb615d92(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:24,262 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:24,262 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/A, priority=12, startTime=1731032724145; duration=0sec 2024-11-08T02:25:24,262 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:24,262 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:A 2024-11-08T02:25:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T02:25:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:24,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:24,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742445_1621 (size=12454) 2024-11-08T02:25:24,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032784312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:24,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:24,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032784418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:24,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032784627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:24,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:24,666 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/d74f13ca4b3b41a1a3cef8246a5bb1f5 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/d74f13ca4b3b41a1a3cef8246a5bb1f5 2024-11-08T02:25:24,671 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/C of 7fa5796b2a900c52ba2e3fc96ec46b12 into d74f13ca4b3b41a1a3cef8246a5bb1f5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:24,671 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:24,671 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/C, priority=12, startTime=1731032724145; duration=0sec 2024-11-08T02:25:24,671 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:24,671 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:C 2024-11-08T02:25:24,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:24,690 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108286a711e7fc14d0fb918eae79d6f9a65_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108286a711e7fc14d0fb918eae79d6f9a65_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:24,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/af6292af44ea4b45b8acd90c894c00dd, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:24,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/af6292af44ea4b45b8acd90c894c00dd is 175, key is test_row_0/A:col10/1731032722895/Put/seqid=0 2024-11-08T02:25:24,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742446_1622 (size=31255) 2024-11-08T02:25:24,698 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/af6292af44ea4b45b8acd90c894c00dd 2024-11-08T02:25:24,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/46a038d11fe94a1b87cb3b2b2b5bf083 is 50, key is test_row_0/B:col10/1731032722895/Put/seqid=0 2024-11-08T02:25:24,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742447_1623 (size=12301) 2024-11-08T02:25:24,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032784938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:25,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032785048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:25,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032785048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:25,049 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 7fa5796b2a900c52ba2e3fc96ec46b12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:25:25,049 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 7fa5796b2a900c52ba2e3fc96ec46b12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:25:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:25,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032785058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:25,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032785058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:25,059 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 7fa5796b2a900c52ba2e3fc96ec46b12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:25:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:25,060 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 7fa5796b2a900c52ba2e3fc96ec46b12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:25:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:25,111 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/46a038d11fe94a1b87cb3b2b2b5bf083 2024-11-08T02:25:25,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/d19dcee4478d4041946c44e7331837b8 is 50, key is test_row_0/C:col10/1731032722895/Put/seqid=0 2024-11-08T02:25:25,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742448_1624 (size=12301) 2024-11-08T02:25:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T02:25:25,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032785448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:25,530 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/d19dcee4478d4041946c44e7331837b8 2024-11-08T02:25:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/af6292af44ea4b45b8acd90c894c00dd as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/af6292af44ea4b45b8acd90c894c00dd 2024-11-08T02:25:25,537 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/af6292af44ea4b45b8acd90c894c00dd, entries=150, sequenceid=353, filesize=30.5 K 2024-11-08T02:25:25,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/46a038d11fe94a1b87cb3b2b2b5bf083 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/46a038d11fe94a1b87cb3b2b2b5bf083 2024-11-08T02:25:25,540 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/46a038d11fe94a1b87cb3b2b2b5bf083, entries=150, sequenceid=353, filesize=12.0 K 2024-11-08T02:25:25,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/d19dcee4478d4041946c44e7331837b8 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/d19dcee4478d4041946c44e7331837b8 2024-11-08T02:25:25,547 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/d19dcee4478d4041946c44e7331837b8, entries=150, sequenceid=353, filesize=12.0 K 2024-11-08T02:25:25,548 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 1340ms, sequenceid=353, compaction requested=false 2024-11-08T02:25:25,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:25,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:25,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-08T02:25:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-08T02:25:25,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-08T02:25:25,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4100 sec 2024-11-08T02:25:25,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.4140 sec 2024-11-08T02:25:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:26,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:26,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:25:26,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110826170dc5d833479a8fea8ac7318c24d0_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032726468/Put/seqid=0 2024-11-08T02:25:26,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742449_1625 (size=12454) 2024-11-08T02:25:26,495 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:26,498 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110826170dc5d833479a8fea8ac7318c24d0_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110826170dc5d833479a8fea8ac7318c24d0_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:26,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f90989a7c19a401c91c606b57d7b4fda, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:26,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f90989a7c19a401c91c606b57d7b4fda is 175, key is test_row_0/A:col10/1731032726468/Put/seqid=0 2024-11-08T02:25:26,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742450_1626 (size=31255) 2024-11-08T02:25:26,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032786558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:26,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:26,570 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:25:26,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032786668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:26,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032786878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:26,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:26,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f90989a7c19a401c91c606b57d7b4fda 2024-11-08T02:25:26,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/63f38570a2b643a4a9cf2f7cb7af2a85 is 50, key is test_row_0/B:col10/1731032726468/Put/seqid=0 2024-11-08T02:25:26,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742451_1627 (size=12301) 2024-11-08T02:25:27,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38302 deadline: 1731032787057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:27,061 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 7fa5796b2a900c52ba2e3fc96ec46b12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:25:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:27,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38320 deadline: 1731032787068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:27,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1731032787069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:27,071 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_0' in 7fa5796b2a900c52ba2e3fc96ec46b12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:25:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:27,071 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_2' in 7fa5796b2a900c52ba2e3fc96ec46b12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:25:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:27,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1731032787077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:27,079 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 331c7316141f,34325,1731032548147 for 'test_row_1' in 7fa5796b2a900c52ba2e3fc96ec46b12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T02:25:27,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:27,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032787187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T02:25:27,308 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:27,309 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:25:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-08T02:25:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T02:25:27,312 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:25:27,313 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:25:27,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:25:27,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/63f38570a2b643a4a9cf2f7cb7af2a85 2024-11-08T02:25:27,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/eb6b7c3756214fc49cfd8a4ec273ac68 is 50, key is test_row_0/C:col10/1731032726468/Put/seqid=0 2024-11-08T02:25:27,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742452_1628 (size=12301) 2024-11-08T02:25:27,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/eb6b7c3756214fc49cfd8a4ec273ac68 2024-11-08T02:25:27,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f90989a7c19a401c91c606b57d7b4fda as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f90989a7c19a401c91c606b57d7b4fda 2024-11-08T02:25:27,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f90989a7c19a401c91c606b57d7b4fda, entries=150, sequenceid=368, filesize=30.5 K 2024-11-08T02:25:27,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/63f38570a2b643a4a9cf2f7cb7af2a85 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/63f38570a2b643a4a9cf2f7cb7af2a85 2024-11-08T02:25:27,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/63f38570a2b643a4a9cf2f7cb7af2a85, entries=150, sequenceid=368, filesize=12.0 K 2024-11-08T02:25:27,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/eb6b7c3756214fc49cfd8a4ec273ac68 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/eb6b7c3756214fc49cfd8a4ec273ac68 2024-11-08T02:25:27,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/eb6b7c3756214fc49cfd8a4ec273ac68, entries=150, sequenceid=368, filesize=12.0 K 2024-11-08T02:25:27,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 925ms, sequenceid=368, compaction requested=true 2024-11-08T02:25:27,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:27,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:25:27,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:27,395 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:27,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:25:27,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:27,395 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:27,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:27,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:27,396 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:27,396 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/A is initiating minor compaction (all files) 2024-11-08T02:25:27,396 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:27,396 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/B is initiating minor compaction (all files) 2024-11-08T02:25:27,396 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/A in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:27,396 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/B in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:27,396 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/34681517e9494107a0984febfb615d92, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/af6292af44ea4b45b8acd90c894c00dd, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f90989a7c19a401c91c606b57d7b4fda] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=92.3 K 2024-11-08T02:25:27,396 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7ca212d7eb0e4229901edeeb58eabf75, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/46a038d11fe94a1b87cb3b2b2b5bf083, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/63f38570a2b643a4a9cf2f7cb7af2a85] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=36.8 K 2024-11-08T02:25:27,396 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:27,396 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/34681517e9494107a0984febfb615d92, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/af6292af44ea4b45b8acd90c894c00dd, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f90989a7c19a401c91c606b57d7b4fda] 2024-11-08T02:25:27,397 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca212d7eb0e4229901edeeb58eabf75, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731032722860 2024-11-08T02:25:27,397 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34681517e9494107a0984febfb615d92, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731032722860 2024-11-08T02:25:27,397 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 46a038d11fe94a1b87cb3b2b2b5bf083, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731032722895 2024-11-08T02:25:27,397 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting af6292af44ea4b45b8acd90c894c00dd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731032722895 2024-11-08T02:25:27,397 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 63f38570a2b643a4a9cf2f7cb7af2a85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731032724293 2024-11-08T02:25:27,397 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting f90989a7c19a401c91c606b57d7b4fda, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731032724293 2024-11-08T02:25:27,417 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:27,418 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#B#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T02:25:27,418 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/c9c82299535b4b3d864c48392dcfba7b is 50, key is test_row_0/B:col10/1731032726468/Put/seqid=0 2024-11-08T02:25:27,444 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411080ecad065581b476ab0dc1cfca52aacfe_7fa5796b2a900c52ba2e3fc96ec46b12 store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:27,446 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411080ecad065581b476ab0dc1cfca52aacfe_7fa5796b2a900c52ba2e3fc96ec46b12, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:27,446 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080ecad065581b476ab0dc1cfca52aacfe_7fa5796b2a900c52ba2e3fc96ec46b12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:27,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34325 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-08T02:25:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:27,468 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:25:27,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742453_1629 (size=13187) 2024-11-08T02:25:27,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742454_1630 (size=4469) 2024-11-08T02:25:27,514 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#A#compaction#528 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:27,515 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/eae1a62389454bacbfb0e131785475cc is 175, key is test_row_0/A:col10/1731032726468/Put/seqid=0 2024-11-08T02:25:27,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a469fa8d3ad24067b81759dcee4790c6_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032726556/Put/seqid=0 2024-11-08T02:25:27,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742455_1631 (size=32141) 2024-11-08T02:25:27,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742456_1632 (size=12454) 2024-11-08T02:25:27,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:27,603 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a469fa8d3ad24067b81759dcee4790c6_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a469fa8d3ad24067b81759dcee4790c6_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:27,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/b613d7d15d444cf9a2d7732ffb147b17, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:27,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/b613d7d15d444cf9a2d7732ffb147b17 is 175, key is test_row_0/A:col10/1731032726556/Put/seqid=0 2024-11-08T02:25:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T02:25:27,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742457_1633 (size=31255) 2024-11-08T02:25:27,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. as already flushing 2024-11-08T02:25:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:27,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032787747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:27,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032787857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:27,889 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/c9c82299535b4b3d864c48392dcfba7b as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c9c82299535b4b3d864c48392dcfba7b 2024-11-08T02:25:27,892 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/B of 7fa5796b2a900c52ba2e3fc96ec46b12 into c9c82299535b4b3d864c48392dcfba7b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:27,892 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:27,892 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/B, priority=13, startTime=1731032727395; duration=0sec 2024-11-08T02:25:27,892 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:27,892 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:B 2024-11-08T02:25:27,892 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:27,893 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:27,893 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/C is initiating minor compaction (all files) 2024-11-08T02:25:27,893 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/C in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:27,893 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/d74f13ca4b3b41a1a3cef8246a5bb1f5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/d19dcee4478d4041946c44e7331837b8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/eb6b7c3756214fc49cfd8a4ec273ac68] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=36.8 K 2024-11-08T02:25:27,893 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting d74f13ca4b3b41a1a3cef8246a5bb1f5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731032722860 2024-11-08T02:25:27,893 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting d19dcee4478d4041946c44e7331837b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731032722895 2024-11-08T02:25:27,894 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting eb6b7c3756214fc49cfd8a4ec273ac68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731032724293 2024-11-08T02:25:27,898 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#C#compaction#530 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:27,898 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/fa8be44bdd19414f9cc1fc1952c612f0 is 50, key is test_row_0/C:col10/1731032726468/Put/seqid=0 2024-11-08T02:25:27,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742458_1634 (size=13153) 2024-11-08T02:25:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T02:25:27,972 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/eae1a62389454bacbfb0e131785475cc as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eae1a62389454bacbfb0e131785475cc 2024-11-08T02:25:27,975 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/A of 7fa5796b2a900c52ba2e3fc96ec46b12 into eae1a62389454bacbfb0e131785475cc(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:27,975 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:27,975 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/A, priority=13, startTime=1731032727395; duration=0sec 2024-11-08T02:25:27,975 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:27,975 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:A 2024-11-08T02:25:28,043 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/b613d7d15d444cf9a2d7732ffb147b17 2024-11-08T02:25:28,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/f1d7a2952aa8479786f085823de7bd9c is 50, key is test_row_0/B:col10/1731032726556/Put/seqid=0 2024-11-08T02:25:28,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742459_1635 (size=12301) 2024-11-08T02:25:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032788067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:28,307 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/fa8be44bdd19414f9cc1fc1952c612f0 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/fa8be44bdd19414f9cc1fc1952c612f0 2024-11-08T02:25:28,315 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/C of 7fa5796b2a900c52ba2e3fc96ec46b12 into fa8be44bdd19414f9cc1fc1952c612f0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:28,315 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:28,315 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/C, priority=13, startTime=1731032727395; duration=0sec 2024-11-08T02:25:28,315 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:28,315 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:C 2024-11-08T02:25:28,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T02:25:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34325 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38308 deadline: 1731032788378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 2024-11-08T02:25:28,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 , the old value is region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:28,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T02:25:28,378 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., hostname=331c7316141f,34325,1731032548147, seqNum=5 because the exception is null or not the one we care about 2024-11-08T02:25:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T02:25:28,451 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/f1d7a2952aa8479786f085823de7bd9c 2024-11-08T02:25:28,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/feb65fc828c64c7e8dc73a03ac5dd489 is 50, key is test_row_0/C:col10/1731032726556/Put/seqid=0 2024-11-08T02:25:28,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742460_1636 (size=12301) 2024-11-08T02:25:28,611 INFO [Thread-2363 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2363. 2024-11-08T02:25:28,611 INFO [Thread-2367 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2367. 2024-11-08T02:25:28,611 DEBUG [Thread-2363 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:28,611 DEBUG [Thread-2367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:28,611 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,611 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,611 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,611 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:28,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:28,612 INFO [Thread-2359 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2359. 2024-11-08T02:25:28,612 DEBUG [Thread-2359 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:28,612 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,612 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:28,614 INFO [Thread-2365 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2365. 2024-11-08T02:25:28,614 DEBUG [Thread-2365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:28,614 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,614 INFO [Thread-2361 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2361. 2024-11-08T02:25:28,614 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,614 DEBUG [Thread-2361 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:28,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:28,614 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,614 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:28,861 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/feb65fc828c64c7e8dc73a03ac5dd489 2024-11-08T02:25:28,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/b613d7d15d444cf9a2d7732ffb147b17 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b613d7d15d444cf9a2d7732ffb147b17 2024-11-08T02:25:28,870 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b613d7d15d444cf9a2d7732ffb147b17, entries=150, sequenceid=392, filesize=30.5 K 2024-11-08T02:25:28,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/f1d7a2952aa8479786f085823de7bd9c as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f1d7a2952aa8479786f085823de7bd9c 2024-11-08T02:25:28,875 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f1d7a2952aa8479786f085823de7bd9c, entries=150, sequenceid=392, filesize=12.0 K 2024-11-08T02:25:28,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/feb65fc828c64c7e8dc73a03ac5dd489 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/feb65fc828c64c7e8dc73a03ac5dd489 2024-11-08T02:25:28,879 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/feb65fc828c64c7e8dc73a03ac5dd489, entries=150, sequenceid=392, filesize=12.0 K 2024-11-08T02:25:28,880 INFO [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 1413ms, sequenceid=392, compaction requested=false 2024-11-08T02:25:28,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:28,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:28,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/331c7316141f:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-08T02:25:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-08T02:25:28,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-08T02:25:28,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-11-08T02:25:28,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.5760 sec 2024-11-08T02:25:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34325 {}] regionserver.HRegion(8855): Flush requested on 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:28,889 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T02:25:28,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T02:25:28,889 INFO [Thread-2350 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2350. 2024-11-08T02:25:28,889 DEBUG [Thread-2350 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:28,889 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,889 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:28,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:28,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b66857da978e4e0cb8a40ca8a31a0c71_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032728888/Put/seqid=0 2024-11-08T02:25:28,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742461_1637 (size=12454) 2024-11-08T02:25:29,300 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:29,305 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b66857da978e4e0cb8a40ca8a31a0c71_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b66857da978e4e0cb8a40ca8a31a0c71_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:29,306 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f0051607f4b144c3b659b91091073472, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:29,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f0051607f4b144c3b659b91091073472 is 175, key is test_row_0/A:col10/1731032728888/Put/seqid=0 2024-11-08T02:25:29,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742462_1638 (size=31255) 2024-11-08T02:25:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T02:25:29,458 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:29,711 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=408, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f0051607f4b144c3b659b91091073472 2024-11-08T02:25:29,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/917cab79ac0942f1b7dae3f7992bf2c3 is 50, key is test_row_0/B:col10/1731032728888/Put/seqid=0 2024-11-08T02:25:29,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742463_1639 (size=12301) 2024-11-08T02:25:30,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/917cab79ac0942f1b7dae3f7992bf2c3 2024-11-08T02:25:30,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/09d0a2b5712242c289b331c20da54926 is 50, key is test_row_0/C:col10/1731032728888/Put/seqid=0 2024-11-08T02:25:30,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742464_1640 (size=12301) 2024-11-08T02:25:30,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/09d0a2b5712242c289b331c20da54926 2024-11-08T02:25:30,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/f0051607f4b144c3b659b91091073472 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f0051607f4b144c3b659b91091073472 2024-11-08T02:25:30,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f0051607f4b144c3b659b91091073472, entries=150, sequenceid=408, filesize=30.5 K 2024-11-08T02:25:30,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/917cab79ac0942f1b7dae3f7992bf2c3 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/917cab79ac0942f1b7dae3f7992bf2c3 2024-11-08T02:25:30,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/917cab79ac0942f1b7dae3f7992bf2c3, entries=150, sequenceid=408, filesize=12.0 K 2024-11-08T02:25:30,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/09d0a2b5712242c289b331c20da54926 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/09d0a2b5712242c289b331c20da54926 2024-11-08T02:25:30,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/09d0a2b5712242c289b331c20da54926, entries=150, sequenceid=408, filesize=12.0 K 2024-11-08T02:25:30,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 1650ms, sequenceid=408, compaction requested=true 2024-11-08T02:25:30,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:30,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:25:30,540 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:30,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:30,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:25:30,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:30,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7fa5796b2a900c52ba2e3fc96ec46b12:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:25:30,540 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:30,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:30,540 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:30,540 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:30,540 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/B is initiating minor compaction (all files) 2024-11-08T02:25:30,540 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/A is initiating minor compaction (all files) 2024-11-08T02:25:30,540 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/B in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:30,540 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/A in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:30,541 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eae1a62389454bacbfb0e131785475cc, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b613d7d15d444cf9a2d7732ffb147b17, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f0051607f4b144c3b659b91091073472] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=92.4 K 2024-11-08T02:25:30,541 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c9c82299535b4b3d864c48392dcfba7b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f1d7a2952aa8479786f085823de7bd9c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/917cab79ac0942f1b7dae3f7992bf2c3] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=36.9 K 2024-11-08T02:25:30,541 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:30,541 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. files: [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eae1a62389454bacbfb0e131785475cc, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b613d7d15d444cf9a2d7732ffb147b17, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f0051607f4b144c3b659b91091073472] 2024-11-08T02:25:30,541 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting c9c82299535b4b3d864c48392dcfba7b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731032724293 2024-11-08T02:25:30,541 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting eae1a62389454bacbfb0e131785475cc, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731032724293 2024-11-08T02:25:30,541 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting f1d7a2952aa8479786f085823de7bd9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731032726543 2024-11-08T02:25:30,541 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting b613d7d15d444cf9a2d7732ffb147b17, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731032726543 2024-11-08T02:25:30,541 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 917cab79ac0942f1b7dae3f7992bf2c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731032727717 2024-11-08T02:25:30,541 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0051607f4b144c3b659b91091073472, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731032727717 2024-11-08T02:25:30,546 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:30,546 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#B#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:30,546 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/35a7e3e10f534737a8eae4d5739c828a is 50, key is test_row_0/B:col10/1731032728888/Put/seqid=0 2024-11-08T02:25:30,547 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110846e10a6fe9ba4373917206c5924c0271_7fa5796b2a900c52ba2e3fc96ec46b12 store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:30,550 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110846e10a6fe9ba4373917206c5924c0271_7fa5796b2a900c52ba2e3fc96ec46b12, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:30,550 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110846e10a6fe9ba4373917206c5924c0271_7fa5796b2a900c52ba2e3fc96ec46b12 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:30,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742465_1641 (size=13289) 2024-11-08T02:25:30,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742466_1642 (size=4469) 2024-11-08T02:25:30,957 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#A#compaction#537 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:30,957 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/35a7e3e10f534737a8eae4d5739c828a as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/35a7e3e10f534737a8eae4d5739c828a 2024-11-08T02:25:30,957 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/e644111ea4174968a6131681f3135ce9 is 175, key is test_row_0/A:col10/1731032728888/Put/seqid=0 2024-11-08T02:25:30,961 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/B of 7fa5796b2a900c52ba2e3fc96ec46b12 into 35a7e3e10f534737a8eae4d5739c828a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:30,961 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:30,961 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/B, priority=13, startTime=1731032730540; duration=0sec 2024-11-08T02:25:30,961 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:25:30,961 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:B 2024-11-08T02:25:30,962 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:25:30,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742467_1643 (size=32243) 2024-11-08T02:25:30,965 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:25:30,965 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1541): 7fa5796b2a900c52ba2e3fc96ec46b12/C is initiating minor compaction (all files) 2024-11-08T02:25:30,965 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7fa5796b2a900c52ba2e3fc96ec46b12/C in TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:30,965 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/fa8be44bdd19414f9cc1fc1952c612f0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/feb65fc828c64c7e8dc73a03ac5dd489, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/09d0a2b5712242c289b331c20da54926] into tmpdir=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp, totalSize=36.9 K 2024-11-08T02:25:30,965 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting fa8be44bdd19414f9cc1fc1952c612f0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731032724293 2024-11-08T02:25:30,965 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting feb65fc828c64c7e8dc73a03ac5dd489, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731032726543 2024-11-08T02:25:30,966 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d0a2b5712242c289b331c20da54926, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731032727717 2024-11-08T02:25:30,967 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/e644111ea4174968a6131681f3135ce9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/e644111ea4174968a6131681f3135ce9 2024-11-08T02:25:30,971 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/A of 7fa5796b2a900c52ba2e3fc96ec46b12 into e644111ea4174968a6131681f3135ce9(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:30,971 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:30,971 INFO [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/A, priority=13, startTime=1731032730540; duration=0sec 2024-11-08T02:25:30,971 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:30,971 DEBUG [RS:0;331c7316141f:34325-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:A 2024-11-08T02:25:30,971 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7fa5796b2a900c52ba2e3fc96ec46b12#C#compaction#538 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:25:30,971 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/b4e4e9154c364a55b7190f58d2ac9b94 is 50, key is test_row_0/C:col10/1731032728888/Put/seqid=0 2024-11-08T02:25:30,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742468_1644 (size=13255) 2024-11-08T02:25:31,069 INFO [Thread-2356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2356. 2024-11-08T02:25:31,069 DEBUG [Thread-2356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:31,069 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:31,069 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:31,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:31,099 INFO [Thread-2354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2354. 2024-11-08T02:25:31,099 DEBUG [Thread-2354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:31,099 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:31,099 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:31,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:31,119 INFO [Thread-2348 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2348. 2024-11-08T02:25:31,119 DEBUG [Thread-2348 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:31,119 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:31,119 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:31,119 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:31,129 INFO [Thread-2352 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2352. 2024-11-08T02:25:31,129 DEBUG [Thread-2352 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T02:25:31,129 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:31,129 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:31,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4017 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3910 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3745 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4036 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3927 2024-11-08T02:25:31,129 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:25:31,129 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T02:25:31,129 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:25:31,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:31,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:31,130 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:31,130 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:25:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-08T02:25:31,132 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032731132"}]},"ts":"1731032731132"} 2024-11-08T02:25:31,133 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:25:31,133 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:25:31,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:25:31,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, UNASSIGN}] 2024-11-08T02:25:31,135 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, UNASSIGN 2024-11-08T02:25:31,136 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=7fa5796b2a900c52ba2e3fc96ec46b12, regionState=CLOSING, regionLocation=331c7316141f,34325,1731032548147 2024-11-08T02:25:31,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, UNASSIGN because future has completed 2024-11-08T02:25:31,137 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:25:31,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147}] 2024-11-08T02:25:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-08T02:25:31,289 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:31,289 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T02:25:31,289 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 7fa5796b2a900c52ba2e3fc96ec46b12, disabling compactions & flushes 2024-11-08T02:25:31,289 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:31,377 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/b4e4e9154c364a55b7190f58d2ac9b94 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/b4e4e9154c364a55b7190f58d2ac9b94 2024-11-08T02:25:31,381 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7fa5796b2a900c52ba2e3fc96ec46b12/C of 7fa5796b2a900c52ba2e3fc96ec46b12 into b4e4e9154c364a55b7190f58d2ac9b94(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:25:31,381 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7fa5796b2a900c52ba2e3fc96ec46b12: 2024-11-08T02:25:31,381 INFO [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., storeName=7fa5796b2a900c52ba2e3fc96ec46b12/C, priority=13, startTime=1731032730540; duration=0sec 2024-11-08T02:25:31,381 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:31,381 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:25:31,381 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:31,381 DEBUG [RS:0;331c7316141f:34325-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7fa5796b2a900c52ba2e3fc96ec46b12:C 2024-11-08T02:25:31,381 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. after waiting 0 ms 2024-11-08T02:25:31,381 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:31,381 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 7fa5796b2a900c52ba2e3fc96ec46b12 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T02:25:31,385 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c8321a447f9f4f92a689225692a6396e_7fa5796b2a900c52ba2e3fc96ec46b12 is 50, key is test_row_0/A:col10/1731032731098/Put/seqid=0 2024-11-08T02:25:31,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742469_1645 (size=12454) 2024-11-08T02:25:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-08T02:25:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-08T02:25:31,788 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:25:31,791 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c8321a447f9f4f92a689225692a6396e_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c8321a447f9f4f92a689225692a6396e_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:31,792 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/b4842433c8954bcd81060d26ae89668f, store: [table=TestAcidGuarantees family=A region=7fa5796b2a900c52ba2e3fc96ec46b12] 2024-11-08T02:25:31,792 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/b4842433c8954bcd81060d26ae89668f is 175, key is test_row_0/A:col10/1731032731098/Put/seqid=0 2024-11-08T02:25:31,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742470_1646 (size=31255) 2024-11-08T02:25:32,195 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/b4842433c8954bcd81060d26ae89668f 2024-11-08T02:25:32,200 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/99bb05cc28a440c5aafd4786e6fe8431 is 50, key is test_row_0/B:col10/1731032731098/Put/seqid=0 2024-11-08T02:25:32,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742471_1647 (size=12301) 2024-11-08T02:25:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-08T02:25:32,604 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/99bb05cc28a440c5aafd4786e6fe8431 2024-11-08T02:25:32,614 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/907018ad59de4b8db1c535a5bec93d4e is 50, key is test_row_0/C:col10/1731032731098/Put/seqid=0 2024-11-08T02:25:32,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742472_1648 (size=12301) 2024-11-08T02:25:33,020 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/907018ad59de4b8db1c535a5bec93d4e 2024-11-08T02:25:33,027 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/A/b4842433c8954bcd81060d26ae89668f as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b4842433c8954bcd81060d26ae89668f 2024-11-08T02:25:33,031 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b4842433c8954bcd81060d26ae89668f, entries=150, sequenceid=418, filesize=30.5 K 2024-11-08T02:25:33,032 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/B/99bb05cc28a440c5aafd4786e6fe8431 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/99bb05cc28a440c5aafd4786e6fe8431 2024-11-08T02:25:33,035 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/99bb05cc28a440c5aafd4786e6fe8431, entries=150, sequenceid=418, filesize=12.0 K 2024-11-08T02:25:33,035 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/.tmp/C/907018ad59de4b8db1c535a5bec93d4e as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/907018ad59de4b8db1c535a5bec93d4e 2024-11-08T02:25:33,037 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/907018ad59de4b8db1c535a5bec93d4e, entries=150, sequenceid=418, filesize=12.0 K 2024-11-08T02:25:33,038 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 1657ms, sequenceid=418, compaction requested=false 2024-11-08T02:25:33,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/474e3a76e42e4a1c944f7c4c66bb437c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/831d7cb1f9c6451b9648a38781b3d1fc, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2799698c910f40f99ec0ab41508bc80e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a2dcacf290ea419d81334a8b2bbe56c8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d9bdce40c7f14f4fa3c3546b23c57182, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a0e3c3b12ecc49cfb283d392a005348e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9413832df9d4474cb0e864b3b74262af, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/20aa8ccdb3f04f5485a0f602e2c85d8d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2b9fc1ac28a34d2c9f7f3794bc049fba, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/13c9b8a5f5194e5ebbce5e0de5264a42, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f5ac7426fcf04146aa893d0d6ffdbdd0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9d4571fb48c544beb2c8628666d789bd, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b0cabf49df3b40bdab49c1b60cf11ef3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d81247b96dbd43f9a81b9fdc2c031516, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1a818718f1e2473d9b4eb06aa722a602, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3aa81feb7fff43ada21f880b6d699b90, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3dcbaa40447f477592ddc419b48e3656, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/151fecc1c75a4866bc01e1dda69e5cc3, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/982143b5de4a4a328f5d865763a72d32, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1d70b0e75802453f8b71ea495051af99, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eb07ef3e3e854264a3166e0080ee8c00, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/48a96b2caa8541288ecd30c7d97be223, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/e27ead74d1194f14ab01044c30881c00, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/34681517e9494107a0984febfb615d92, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/af6292af44ea4b45b8acd90c894c00dd, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eae1a62389454bacbfb0e131785475cc, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f90989a7c19a401c91c606b57d7b4fda, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b613d7d15d444cf9a2d7732ffb147b17, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f0051607f4b144c3b659b91091073472] to archive 2024-11-08T02:25:33,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:25:33,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/474e3a76e42e4a1c944f7c4c66bb437c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/474e3a76e42e4a1c944f7c4c66bb437c 2024-11-08T02:25:33,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/831d7cb1f9c6451b9648a38781b3d1fc to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/831d7cb1f9c6451b9648a38781b3d1fc 2024-11-08T02:25:33,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2799698c910f40f99ec0ab41508bc80e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2799698c910f40f99ec0ab41508bc80e 2024-11-08T02:25:33,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a2dcacf290ea419d81334a8b2bbe56c8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a2dcacf290ea419d81334a8b2bbe56c8 2024-11-08T02:25:33,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d9bdce40c7f14f4fa3c3546b23c57182 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d9bdce40c7f14f4fa3c3546b23c57182 2024-11-08T02:25:33,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a0e3c3b12ecc49cfb283d392a005348e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/a0e3c3b12ecc49cfb283d392a005348e 2024-11-08T02:25:33,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9413832df9d4474cb0e864b3b74262af to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9413832df9d4474cb0e864b3b74262af 2024-11-08T02:25:33,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/20aa8ccdb3f04f5485a0f602e2c85d8d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/20aa8ccdb3f04f5485a0f602e2c85d8d 2024-11-08T02:25:33,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2b9fc1ac28a34d2c9f7f3794bc049fba to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/2b9fc1ac28a34d2c9f7f3794bc049fba 2024-11-08T02:25:33,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/13c9b8a5f5194e5ebbce5e0de5264a42 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/13c9b8a5f5194e5ebbce5e0de5264a42 2024-11-08T02:25:33,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f5ac7426fcf04146aa893d0d6ffdbdd0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f5ac7426fcf04146aa893d0d6ffdbdd0 2024-11-08T02:25:33,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9d4571fb48c544beb2c8628666d789bd to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/9d4571fb48c544beb2c8628666d789bd 2024-11-08T02:25:33,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b0cabf49df3b40bdab49c1b60cf11ef3 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b0cabf49df3b40bdab49c1b60cf11ef3 2024-11-08T02:25:33,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d81247b96dbd43f9a81b9fdc2c031516 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/d81247b96dbd43f9a81b9fdc2c031516 2024-11-08T02:25:33,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1a818718f1e2473d9b4eb06aa722a602 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1a818718f1e2473d9b4eb06aa722a602 2024-11-08T02:25:33,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3aa81feb7fff43ada21f880b6d699b90 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3aa81feb7fff43ada21f880b6d699b90 2024-11-08T02:25:33,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3dcbaa40447f477592ddc419b48e3656 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/3dcbaa40447f477592ddc419b48e3656 2024-11-08T02:25:33,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/151fecc1c75a4866bc01e1dda69e5cc3 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/151fecc1c75a4866bc01e1dda69e5cc3 2024-11-08T02:25:33,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/982143b5de4a4a328f5d865763a72d32 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/982143b5de4a4a328f5d865763a72d32 2024-11-08T02:25:33,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1d70b0e75802453f8b71ea495051af99 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/1d70b0e75802453f8b71ea495051af99 2024-11-08T02:25:33,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eb07ef3e3e854264a3166e0080ee8c00 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eb07ef3e3e854264a3166e0080ee8c00 2024-11-08T02:25:33,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/48a96b2caa8541288ecd30c7d97be223 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/48a96b2caa8541288ecd30c7d97be223 2024-11-08T02:25:33,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/e27ead74d1194f14ab01044c30881c00 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/e27ead74d1194f14ab01044c30881c00 2024-11-08T02:25:33,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/34681517e9494107a0984febfb615d92 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/34681517e9494107a0984febfb615d92 2024-11-08T02:25:33,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/af6292af44ea4b45b8acd90c894c00dd to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/af6292af44ea4b45b8acd90c894c00dd 2024-11-08T02:25:33,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eae1a62389454bacbfb0e131785475cc to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/eae1a62389454bacbfb0e131785475cc 2024-11-08T02:25:33,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f90989a7c19a401c91c606b57d7b4fda to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f90989a7c19a401c91c606b57d7b4fda 2024-11-08T02:25:33,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b613d7d15d444cf9a2d7732ffb147b17 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b613d7d15d444cf9a2d7732ffb147b17 2024-11-08T02:25:33,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f0051607f4b144c3b659b91091073472 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/f0051607f4b144c3b659b91091073472 2024-11-08T02:25:33,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4a421d0dedbc4364bd956d6f1ad48842, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/56f9d326ea7f44cba49e8599eb08d0e4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/fee1fd58922d49bd938b5f07d06b1763, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c152612366d44f84aa6d6b452307afb2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/25094e2e483b4d49b60a6837d8c2b49c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/d94bf9fbe92e43a7b09a52eb42706c21, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7469dcc32da54b0b9f487d5d4df1a0bd, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a10be4f4de81486da1e23f2caf3a777f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/56fcc5438c6845598a2810fa66246209, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/997ffbad2c084417bca2aaf77b8a18a6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/846caddb279145caa09c2537dba845a7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/0ee1e38f043b46fdbebbb6b810194b83, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f195db62591b4bbd94832bfe6434be33, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/d0a807625c58410d9b1ed265c8755aa4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7ad462d075174f4d96ec76f7c00cad90, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/3ddb4292b36142609a4050733c0b3fff, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4e24fc096aa5490eb97a711630c60f1a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/705f211f404546a88bd7bcc91bd8aed6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/01e18519853d455997384c934f6826d4, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a386760b162544d9b9cc57afe3b91269, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a363ce5a6654420385783ae71c5f0a78, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a22c3014f5654f4294313208d6b51dec, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7ca212d7eb0e4229901edeeb58eabf75, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c51e06d98cbc4a359d2b0b7cf17c7bb6, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/46a038d11fe94a1b87cb3b2b2b5bf083, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c9c82299535b4b3d864c48392dcfba7b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/63f38570a2b643a4a9cf2f7cb7af2a85, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f1d7a2952aa8479786f085823de7bd9c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/917cab79ac0942f1b7dae3f7992bf2c3] to archive 2024-11-08T02:25:33,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:25:33,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4a421d0dedbc4364bd956d6f1ad48842 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4a421d0dedbc4364bd956d6f1ad48842 2024-11-08T02:25:33,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/56f9d326ea7f44cba49e8599eb08d0e4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/56f9d326ea7f44cba49e8599eb08d0e4 2024-11-08T02:25:33,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/fee1fd58922d49bd938b5f07d06b1763 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/fee1fd58922d49bd938b5f07d06b1763 2024-11-08T02:25:33,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c152612366d44f84aa6d6b452307afb2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c152612366d44f84aa6d6b452307afb2 2024-11-08T02:25:33,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/25094e2e483b4d49b60a6837d8c2b49c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/25094e2e483b4d49b60a6837d8c2b49c 2024-11-08T02:25:33,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/d94bf9fbe92e43a7b09a52eb42706c21 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/d94bf9fbe92e43a7b09a52eb42706c21 2024-11-08T02:25:33,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7469dcc32da54b0b9f487d5d4df1a0bd to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7469dcc32da54b0b9f487d5d4df1a0bd 2024-11-08T02:25:33,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a10be4f4de81486da1e23f2caf3a777f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a10be4f4de81486da1e23f2caf3a777f 2024-11-08T02:25:33,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/56fcc5438c6845598a2810fa66246209 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/56fcc5438c6845598a2810fa66246209 2024-11-08T02:25:33,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/997ffbad2c084417bca2aaf77b8a18a6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/997ffbad2c084417bca2aaf77b8a18a6 2024-11-08T02:25:33,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/846caddb279145caa09c2537dba845a7 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/846caddb279145caa09c2537dba845a7 2024-11-08T02:25:33,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/0ee1e38f043b46fdbebbb6b810194b83 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/0ee1e38f043b46fdbebbb6b810194b83 2024-11-08T02:25:33,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f195db62591b4bbd94832bfe6434be33 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f195db62591b4bbd94832bfe6434be33 2024-11-08T02:25:33,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/d0a807625c58410d9b1ed265c8755aa4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/d0a807625c58410d9b1ed265c8755aa4 2024-11-08T02:25:33,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7ad462d075174f4d96ec76f7c00cad90 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7ad462d075174f4d96ec76f7c00cad90 2024-11-08T02:25:33,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/3ddb4292b36142609a4050733c0b3fff to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/3ddb4292b36142609a4050733c0b3fff 2024-11-08T02:25:33,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4e24fc096aa5490eb97a711630c60f1a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/4e24fc096aa5490eb97a711630c60f1a 2024-11-08T02:25:33,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/705f211f404546a88bd7bcc91bd8aed6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/705f211f404546a88bd7bcc91bd8aed6 2024-11-08T02:25:33,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/01e18519853d455997384c934f6826d4 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/01e18519853d455997384c934f6826d4 2024-11-08T02:25:33,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a386760b162544d9b9cc57afe3b91269 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a386760b162544d9b9cc57afe3b91269 2024-11-08T02:25:33,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a363ce5a6654420385783ae71c5f0a78 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a363ce5a6654420385783ae71c5f0a78 2024-11-08T02:25:33,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a22c3014f5654f4294313208d6b51dec to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/a22c3014f5654f4294313208d6b51dec 2024-11-08T02:25:33,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7ca212d7eb0e4229901edeeb58eabf75 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/7ca212d7eb0e4229901edeeb58eabf75 2024-11-08T02:25:33,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c51e06d98cbc4a359d2b0b7cf17c7bb6 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c51e06d98cbc4a359d2b0b7cf17c7bb6 2024-11-08T02:25:33,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/46a038d11fe94a1b87cb3b2b2b5bf083 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/46a038d11fe94a1b87cb3b2b2b5bf083 2024-11-08T02:25:33,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c9c82299535b4b3d864c48392dcfba7b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/c9c82299535b4b3d864c48392dcfba7b 2024-11-08T02:25:33,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/63f38570a2b643a4a9cf2f7cb7af2a85 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/63f38570a2b643a4a9cf2f7cb7af2a85 2024-11-08T02:25:33,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f1d7a2952aa8479786f085823de7bd9c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/f1d7a2952aa8479786f085823de7bd9c 2024-11-08T02:25:33,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/917cab79ac0942f1b7dae3f7992bf2c3 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/917cab79ac0942f1b7dae3f7992bf2c3 2024-11-08T02:25:33,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/37ec5c578b8541cc96a1cfc82d2b8119, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/a375570f45b54e8ebf603454b7285e3d, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/b8738b49190a4a309bea85e8a9379425, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/7c06384e8d9b4b78928a68ceb638d93a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/71977075b17349559a60267bad08ab6f, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/e315664a3a734f9ab5591e233b031fd5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/1a2e204c87d54bddbc59c1f7fcfa3928, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/cb4ed301a82b46c38542d0bbea5f8e9b, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/8201f65f702140ccbe4560df9c20bffa, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0736298ff9a349ffabd10e4e673cf9e7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/8f5401a1c6b04c939421a2c8fc5c0b7e, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/fd532c38a06c4b3d9d48c1d8a6428887, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0050c60c737f439497f071be265533f2, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/c2d1b5bfba3b4e61887a22c204ff3475, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0aa1a289ef4041b0a66423fa65dfefed, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/44a1a00f0d484481a0f0c3b58f3fb086, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/854ce21612b94752a3a71e66db330162, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/7d596f6924064f8eb30967904e24bbd7, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/364c937e69bf4cb1953838bf65aecf97, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/2cfeddcac190453badd7d54d0a7d406c, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/6421633ed5624f5993fbe9aca0dee41a, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/d74f13ca4b3b41a1a3cef8246a5bb1f5, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/28638651d4f940d58028561daf83e361, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/d19dcee4478d4041946c44e7331837b8, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/fa8be44bdd19414f9cc1fc1952c612f0, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/eb6b7c3756214fc49cfd8a4ec273ac68, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/feb65fc828c64c7e8dc73a03ac5dd489, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/09d0a2b5712242c289b331c20da54926] to archive 2024-11-08T02:25:33,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:25:33,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/37ec5c578b8541cc96a1cfc82d2b8119 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/37ec5c578b8541cc96a1cfc82d2b8119 2024-11-08T02:25:33,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/a375570f45b54e8ebf603454b7285e3d to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/a375570f45b54e8ebf603454b7285e3d 2024-11-08T02:25:33,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/b8738b49190a4a309bea85e8a9379425 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/b8738b49190a4a309bea85e8a9379425 2024-11-08T02:25:33,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/7c06384e8d9b4b78928a68ceb638d93a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/7c06384e8d9b4b78928a68ceb638d93a 2024-11-08T02:25:33,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/71977075b17349559a60267bad08ab6f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/71977075b17349559a60267bad08ab6f 2024-11-08T02:25:33,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/e315664a3a734f9ab5591e233b031fd5 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/e315664a3a734f9ab5591e233b031fd5 2024-11-08T02:25:33,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/1a2e204c87d54bddbc59c1f7fcfa3928 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/1a2e204c87d54bddbc59c1f7fcfa3928 2024-11-08T02:25:33,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/cb4ed301a82b46c38542d0bbea5f8e9b to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/cb4ed301a82b46c38542d0bbea5f8e9b 2024-11-08T02:25:33,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/8201f65f702140ccbe4560df9c20bffa to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/8201f65f702140ccbe4560df9c20bffa 2024-11-08T02:25:33,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0736298ff9a349ffabd10e4e673cf9e7 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0736298ff9a349ffabd10e4e673cf9e7 2024-11-08T02:25:33,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/8f5401a1c6b04c939421a2c8fc5c0b7e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/8f5401a1c6b04c939421a2c8fc5c0b7e 2024-11-08T02:25:33,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/fd532c38a06c4b3d9d48c1d8a6428887 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/fd532c38a06c4b3d9d48c1d8a6428887 2024-11-08T02:25:33,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0050c60c737f439497f071be265533f2 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0050c60c737f439497f071be265533f2 2024-11-08T02:25:33,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/c2d1b5bfba3b4e61887a22c204ff3475 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/c2d1b5bfba3b4e61887a22c204ff3475 2024-11-08T02:25:33,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0aa1a289ef4041b0a66423fa65dfefed to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/0aa1a289ef4041b0a66423fa65dfefed 2024-11-08T02:25:33,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/44a1a00f0d484481a0f0c3b58f3fb086 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/44a1a00f0d484481a0f0c3b58f3fb086 2024-11-08T02:25:33,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/854ce21612b94752a3a71e66db330162 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/854ce21612b94752a3a71e66db330162 2024-11-08T02:25:33,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/7d596f6924064f8eb30967904e24bbd7 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/7d596f6924064f8eb30967904e24bbd7 2024-11-08T02:25:33,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/364c937e69bf4cb1953838bf65aecf97 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/364c937e69bf4cb1953838bf65aecf97 2024-11-08T02:25:33,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/2cfeddcac190453badd7d54d0a7d406c to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/2cfeddcac190453badd7d54d0a7d406c 2024-11-08T02:25:33,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/6421633ed5624f5993fbe9aca0dee41a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/6421633ed5624f5993fbe9aca0dee41a 2024-11-08T02:25:33,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/d74f13ca4b3b41a1a3cef8246a5bb1f5 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/d74f13ca4b3b41a1a3cef8246a5bb1f5 2024-11-08T02:25:33,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/28638651d4f940d58028561daf83e361 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/28638651d4f940d58028561daf83e361 2024-11-08T02:25:33,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/d19dcee4478d4041946c44e7331837b8 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/d19dcee4478d4041946c44e7331837b8 2024-11-08T02:25:33,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/fa8be44bdd19414f9cc1fc1952c612f0 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/fa8be44bdd19414f9cc1fc1952c612f0 2024-11-08T02:25:33,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/eb6b7c3756214fc49cfd8a4ec273ac68 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/eb6b7c3756214fc49cfd8a4ec273ac68 2024-11-08T02:25:33,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/feb65fc828c64c7e8dc73a03ac5dd489 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/feb65fc828c64c7e8dc73a03ac5dd489 2024-11-08T02:25:33,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/09d0a2b5712242c289b331c20da54926 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/09d0a2b5712242c289b331c20da54926 2024-11-08T02:25:33,101 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/recovered.edits/421.seqid, newMaxSeqId=421, maxSeqId=4 2024-11-08T02:25:33,102 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. 2024-11-08T02:25:33,102 DEBUG [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 7fa5796b2a900c52ba2e3fc96ec46b12: Waiting for close lock at 1731032731289Running coprocessor pre-close hooks at 1731032731289Disabling compacts and flushes for region at 1731032731289Disabling writes for close at 1731032731381 (+92 ms)Obtaining lock to block concurrent updates at 1731032731381Preparing flush snapshotting stores in 7fa5796b2a900c52ba2e3fc96ec46b12 at 1731032731381Finished memstore snapshotting TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731032731381Flushing stores of TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12. at 1731032731381Flushing 7fa5796b2a900c52ba2e3fc96ec46b12/A: creating writer at 1731032731382 (+1 ms)Flushing mob file 7fa5796b2a900c52ba2e3fc96ec46b12/A: appending metadata at 1731032731385 (+3 ms)Flushing mob file 7fa5796b2a900c52ba2e3fc96ec46b12/A: closing flushed file at 1731032731385Flushing 7fa5796b2a900c52ba2e3fc96ec46b12/A: appending metadata at 1731032731792 (+407 ms)Flushing 7fa5796b2a900c52ba2e3fc96ec46b12/A: closing flushed file at 1731032731792Flushing 7fa5796b2a900c52ba2e3fc96ec46b12/B: creating writer at 1731032732198 (+406 ms)Flushing 7fa5796b2a900c52ba2e3fc96ec46b12/B: appending metadata at 1731032732200 (+2 ms)Flushing 7fa5796b2a900c52ba2e3fc96ec46b12/B: closing flushed file at 1731032732200Flushing 7fa5796b2a900c52ba2e3fc96ec46b12/C: creating writer at 1731032732610 (+410 ms)Flushing 7fa5796b2a900c52ba2e3fc96ec46b12/C: appending metadata at 1731032732613 (+3 ms)Flushing 7fa5796b2a900c52ba2e3fc96ec46b12/C: closing flushed file at 1731032732613Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b753bfe: reopening flushed file at 1731032733027 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d821606: reopening flushed file at 1731032733032 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48d4f83c: reopening flushed file at 1731032733035 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 7fa5796b2a900c52ba2e3fc96ec46b12 in 1657ms, sequenceid=418, compaction requested=false at 1731032733038 (+3 ms)Writing region close event to WAL at 1731032733099 (+61 ms)Running coprocessor post-close hooks at 1731032733102 (+3 ms)Closed at 1731032733102 2024-11-08T02:25:33,103 INFO [RS_CLOSE_REGION-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,104 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=7fa5796b2a900c52ba2e3fc96ec46b12, regionState=CLOSED 2024-11-08T02:25:33,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 because future has completed 2024-11-08T02:25:33,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-08T02:25:33,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 7fa5796b2a900c52ba2e3fc96ec46b12, server=331c7316141f,34325,1731032548147 in 1.9680 sec 2024-11-08T02:25:33,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-08T02:25:33,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7fa5796b2a900c52ba2e3fc96ec46b12, UNASSIGN in 1.9720 sec 2024-11-08T02:25:33,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-08T02:25:33,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9750 sec 2024-11-08T02:25:33,112 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731032733111"}]},"ts":"1731032733111"} 2024-11-08T02:25:33,113 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:25:33,113 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:25:33,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9840 sec 2024-11-08T02:25:33,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-08T02:25:33,279 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:33,280 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:25:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:33,285 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:33,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-08T02:25:33,287 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:33,290 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,293 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C, FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/recovered.edits] 2024-11-08T02:25:33,297 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b4842433c8954bcd81060d26ae89668f to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/b4842433c8954bcd81060d26ae89668f 2024-11-08T02:25:33,299 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/e644111ea4174968a6131681f3135ce9 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/A/e644111ea4174968a6131681f3135ce9 2024-11-08T02:25:33,303 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/35a7e3e10f534737a8eae4d5739c828a to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/35a7e3e10f534737a8eae4d5739c828a 2024-11-08T02:25:33,305 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/99bb05cc28a440c5aafd4786e6fe8431 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/B/99bb05cc28a440c5aafd4786e6fe8431 2024-11-08T02:25:33,309 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/907018ad59de4b8db1c535a5bec93d4e to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/907018ad59de4b8db1c535a5bec93d4e 2024-11-08T02:25:33,312 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/b4e4e9154c364a55b7190f58d2ac9b94 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/C/b4e4e9154c364a55b7190f58d2ac9b94 2024-11-08T02:25:33,316 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/recovered.edits/421.seqid to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12/recovered.edits/421.seqid 2024-11-08T02:25:33,317 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/default/TestAcidGuarantees/7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,317 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:25:33,317 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:25:33,318 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T02:25:33,323 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108003018248d8d4b5ca919722acff84fb9_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108003018248d8d4b5ca919722acff84fb9_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,324 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108125eafd5330748cd93efd338582e8589_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108125eafd5330748cd93efd338582e8589_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,326 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081cd3105e9e6e43f094a3af0383a1e0cb_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081cd3105e9e6e43f094a3af0383a1e0cb_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,328 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110826170dc5d833479a8fea8ac7318c24d0_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110826170dc5d833479a8fea8ac7318c24d0_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,330 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110827c7a157f013442b9fed784fa7ecf8bb_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110827c7a157f013442b9fed784fa7ecf8bb_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,332 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108286a711e7fc14d0fb918eae79d6f9a65_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108286a711e7fc14d0fb918eae79d6f9a65_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,334 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084b9baefe478946049fdb614dc3e806ef_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084b9baefe478946049fdb614dc3e806ef_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,336 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085e86c066ddee47d5aed6283b8c81b24e_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085e86c066ddee47d5aed6283b8c81b24e_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,337 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110865a8f9d3c7494102978abbef2de6d1fd_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110865a8f9d3c7494102978abbef2de6d1fd_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,339 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086e37403b63884f8090b7102a03bf1214_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086e37403b63884f8090b7102a03bf1214_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,341 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088156d53ed6524161851e1fa33ebb35ac_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088156d53ed6524161851e1fa33ebb35ac_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,343 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a469fa8d3ad24067b81759dcee4790c6_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a469fa8d3ad24067b81759dcee4790c6_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,345 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a90e3d677e634a3f96445f0a71f311ab_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a90e3d677e634a3f96445f0a71f311ab_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,347 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b55b473e419e42e7961db307c0ac693a_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b55b473e419e42e7961db307c0ac693a_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,349 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b66857da978e4e0cb8a40ca8a31a0c71_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b66857da978e4e0cb8a40ca8a31a0c71_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,351 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108baf0a4f3528a4ebbafa374e670709610_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108baf0a4f3528a4ebbafa374e670709610_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,353 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c8321a447f9f4f92a689225692a6396e_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c8321a447f9f4f92a689225692a6396e_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,355 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ca5775f9bef54d6899144ac5a0e8b24b_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ca5775f9bef54d6899144ac5a0e8b24b_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,357 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cee40935b24a472bb304a9847f9b96cc_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cee40935b24a472bb304a9847f9b96cc_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,359 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108dc0e96dbaafb480fb83156996903e435_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108dc0e96dbaafb480fb83156996903e435_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,361 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ec41cc27b22641c1a11a30657f9582fe_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ec41cc27b22641c1a11a30657f9582fe_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,363 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fdd9f750beb34a1e8e13ef818f8a6859_7fa5796b2a900c52ba2e3fc96ec46b12 to hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fdd9f750beb34a1e8e13ef818f8a6859_7fa5796b2a900c52ba2e3fc96ec46b12 2024-11-08T02:25:33,364 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:25:33,368 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:33,371 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:25:33,373 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:25:33,375 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:33,375 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:25:33,375 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731032733375"}]},"ts":"9223372036854775807"} 2024-11-08T02:25:33,377 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T02:25:33,377 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7fa5796b2a900c52ba2e3fc96ec46b12, NAME => 'TestAcidGuarantees,,1731032706524.7fa5796b2a900c52ba2e3fc96ec46b12.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:25:33,378 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:25:33,378 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731032733378"}]},"ts":"9223372036854775807"} 2024-11-08T02:25:33,380 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T02:25:33,381 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:25:33,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 100 msec 2024-11-08T02:25:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-08T02:25:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T02:25:33,398 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T02:25:33,408 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 236) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 425) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6562 (was 6580) 2024-11-08T02:25:33,408 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T02:25:33,408 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T02:25:33,409 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:25:33,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:33,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:33,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T02:25:33,409 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T02:25:33,409 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1372924334, stopped=false 2024-11-08T02:25:33,410 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=331c7316141f,41605,1731032547397 2024-11-08T02:25:33,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T02:25:33,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T02:25:33,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:25:33,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:25:33,506 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T02:25:33,508 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T02:25:33,508 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T02:25:33,508 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T02:25:33,509 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:25:33,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:33,510 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '331c7316141f,34325,1731032548147' ***** 2024-11-08T02:25:33,510 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T02:25:33,510 INFO [RS:0;331c7316141f:34325 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T02:25:33,511 INFO [RS:0;331c7316141f:34325 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T02:25:33,511 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T02:25:33,511 INFO [RS:0;331c7316141f:34325 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T02:25:33,512 INFO [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(959): stopping server 331c7316141f,34325,1731032548147 2024-11-08T02:25:33,512 INFO [RS:0;331c7316141f:34325 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T02:25:33,512 INFO [RS:0;331c7316141f:34325 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;331c7316141f:34325. 2024-11-08T02:25:33,512 DEBUG [RS:0;331c7316141f:34325 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:25:33,512 DEBUG [RS:0;331c7316141f:34325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:33,512 INFO [RS:0;331c7316141f:34325 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T02:25:33,512 INFO [RS:0;331c7316141f:34325 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T02:25:33,512 INFO [RS:0;331c7316141f:34325 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T02:25:33,513 INFO [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T02:25:33,514 INFO [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T02:25:33,514 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T02:25:33,514 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T02:25:33,514 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T02:25:33,514 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T02:25:33,515 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T02:25:33,515 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T02:25:33,515 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T02:25:33,515 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-08T02:25:33,535 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/info/36361aca4fc54f25bb8cdfeea08e55bc is 95, key is TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd./info:/1731032580726/DeleteFamily/seqid=0 2024-11-08T02:25:33,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742473_1649 (size=6514) 2024-11-08T02:25:33,577 INFO [regionserver/331c7316141f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T02:25:33,715 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T02:25:33,915 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T02:25:33,947 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/info/36361aca4fc54f25bb8cdfeea08e55bc 2024-11-08T02:25:33,972 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/ns/45d62c297fe14a0aa32aed3c5afe5bbe is 93, key is TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd./ns:/1731032580688/DeleteFamily/seqid=0 2024-11-08T02:25:33,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742474_1650 (size=5906) 2024-11-08T02:25:34,115 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T02:25:34,263 INFO [regionserver/331c7316141f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T02:25:34,264 INFO [regionserver/331c7316141f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T02:25:34,315 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T02:25:34,377 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/ns/45d62c297fe14a0aa32aed3c5afe5bbe 2024-11-08T02:25:34,404 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/rep_barrier/4daabf3e775147509e1f37c2608070e9 is 102, key is TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd./rep_barrier:/1731032580688/DeleteFamily/seqid=0 2024-11-08T02:25:34,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742475_1651 (size=6025) 2024-11-08T02:25:34,515 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T02:25:34,716 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T02:25:34,808 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/rep_barrier/4daabf3e775147509e1f37c2608070e9 2024-11-08T02:25:34,838 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/table/9d2de818d92c41bd9ebff19a05aa9fbd is 96, key is TestAcidGuarantees,,1731032552485.8469534ae513a00a18db97d665b015cd./table:/1731032580688/DeleteFamily/seqid=0 2024-11-08T02:25:34,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742476_1652 (size=5936) 2024-11-08T02:25:34,916 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T02:25:35,116 DEBUG [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T02:25:35,242 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/table/9d2de818d92c41bd9ebff19a05aa9fbd 2024-11-08T02:25:35,253 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/info/36361aca4fc54f25bb8cdfeea08e55bc as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/info/36361aca4fc54f25bb8cdfeea08e55bc 2024-11-08T02:25:35,256 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/info/36361aca4fc54f25bb8cdfeea08e55bc, entries=12, sequenceid=90, filesize=6.4 K 2024-11-08T02:25:35,257 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/ns/45d62c297fe14a0aa32aed3c5afe5bbe as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/ns/45d62c297fe14a0aa32aed3c5afe5bbe 2024-11-08T02:25:35,261 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/ns/45d62c297fe14a0aa32aed3c5afe5bbe, entries=8, sequenceid=90, filesize=5.8 K 2024-11-08T02:25:35,262 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/rep_barrier/4daabf3e775147509e1f37c2608070e9 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/rep_barrier/4daabf3e775147509e1f37c2608070e9 2024-11-08T02:25:35,265 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/rep_barrier/4daabf3e775147509e1f37c2608070e9, entries=6, sequenceid=90, filesize=5.9 K 2024-11-08T02:25:35,266 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/.tmp/table/9d2de818d92c41bd9ebff19a05aa9fbd as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/table/9d2de818d92c41bd9ebff19a05aa9fbd 2024-11-08T02:25:35,270 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/table/9d2de818d92c41bd9ebff19a05aa9fbd, entries=7, sequenceid=90, filesize=5.8 K 2024-11-08T02:25:35,271 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1756ms, sequenceid=90, compaction requested=false 2024-11-08T02:25:35,276 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-08T02:25:35,277 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T02:25:35,277 INFO [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T02:25:35,277 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731032733514Running coprocessor pre-close hooks at 1731032733514Disabling compacts and flushes for region at 1731032733514Disabling writes for close at 1731032733515 (+1 ms)Obtaining lock to block concurrent updates at 1731032733515Preparing flush snapshotting stores in 1588230740 at 1731032733515Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731032733515Flushing stores of hbase:meta,,1.1588230740 at 1731032733516 (+1 ms)Flushing 1588230740/info: creating writer at 1731032733516Flushing 1588230740/info: appending metadata at 1731032733534 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731032733534Flushing 1588230740/ns: creating writer at 1731032733956 (+422 ms)Flushing 1588230740/ns: appending metadata at 1731032733972 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731032733972Flushing 1588230740/rep_barrier: creating writer at 1731032734387 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731032734404 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731032734404Flushing 1588230740/table: creating writer at 1731032734818 (+414 ms)Flushing 1588230740/table: appending metadata at 1731032734837 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731032734837Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@181fc539: reopening flushed file at 1731032735252 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44338ff2: reopening flushed file at 1731032735257 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78281b23: reopening flushed file at 1731032735261 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d1e2de6: reopening flushed file at 1731032735265 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1756ms, sequenceid=90, compaction requested=false at 1731032735271 (+6 ms)Writing region close event to WAL at 1731032735273 (+2 ms)Running coprocessor post-close hooks at 1731032735277 (+4 ms)Closed at 1731032735277 2024-11-08T02:25:35,277 DEBUG [RS_CLOSE_META-regionserver/331c7316141f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T02:25:35,316 INFO [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(976): stopping server 331c7316141f,34325,1731032548147; all regions closed. 2024-11-08T02:25:35,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741834_1010 (size=25344) 2024-11-08T02:25:35,322 DEBUG [RS:0;331c7316141f:34325 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/oldWALs 2024-11-08T02:25:35,323 INFO [RS:0;331c7316141f:34325 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 331c7316141f%2C34325%2C1731032548147.meta:.meta(num 1731032551915) 2024-11-08T02:25:35,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741833_1009 (size=14604174) 2024-11-08T02:25:35,327 DEBUG [RS:0;331c7316141f:34325 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/oldWALs 2024-11-08T02:25:35,327 INFO [RS:0;331c7316141f:34325 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 331c7316141f%2C34325%2C1731032548147:(num 1731032551421) 2024-11-08T02:25:35,327 DEBUG [RS:0;331c7316141f:34325 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:25:35,327 INFO [RS:0;331c7316141f:34325 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T02:25:35,327 INFO [RS:0;331c7316141f:34325 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T02:25:35,327 INFO [RS:0;331c7316141f:34325 {}] hbase.ChoreService(370): Chore service for: regionserver/331c7316141f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T02:25:35,327 INFO [RS:0;331c7316141f:34325 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T02:25:35,327 INFO [regionserver/331c7316141f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T02:25:35,328 INFO [RS:0;331c7316141f:34325 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34325 2024-11-08T02:25:35,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/331c7316141f,34325,1731032548147 2024-11-08T02:25:35,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T02:25:35,401 INFO [RS:0;331c7316141f:34325 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T02:25:35,413 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [331c7316141f,34325,1731032548147] 2024-11-08T02:25:35,424 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/331c7316141f,34325,1731032548147 already deleted, retry=false 2024-11-08T02:25:35,424 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 331c7316141f,34325,1731032548147 expired; onlineServers=0 2024-11-08T02:25:35,424 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '331c7316141f,41605,1731032547397' ***** 2024-11-08T02:25:35,424 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T02:25:35,425 INFO [M:0;331c7316141f:41605 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T02:25:35,425 INFO [M:0;331c7316141f:41605 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T02:25:35,425 DEBUG [M:0;331c7316141f:41605 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T02:25:35,425 DEBUG [M:0;331c7316141f:41605 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T02:25:35,425 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T02:25:35,425 DEBUG [master/331c7316141f:0:becomeActiveMaster-HFileCleaner.small.0-1731032551116 {}] cleaner.HFileCleaner(306): Exit Thread[master/331c7316141f:0:becomeActiveMaster-HFileCleaner.small.0-1731032551116,5,FailOnTimeoutGroup] 2024-11-08T02:25:35,425 DEBUG [master/331c7316141f:0:becomeActiveMaster-HFileCleaner.large.0-1731032551114 {}] cleaner.HFileCleaner(306): Exit Thread[master/331c7316141f:0:becomeActiveMaster-HFileCleaner.large.0-1731032551114,5,FailOnTimeoutGroup] 2024-11-08T02:25:35,426 INFO [M:0;331c7316141f:41605 {}] hbase.ChoreService(370): Chore service for: master/331c7316141f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T02:25:35,426 INFO [M:0;331c7316141f:41605 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T02:25:35,426 DEBUG [M:0;331c7316141f:41605 {}] master.HMaster(1795): Stopping service threads 2024-11-08T02:25:35,426 INFO [M:0;331c7316141f:41605 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T02:25:35,426 INFO [M:0;331c7316141f:41605 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T02:25:35,426 ERROR [M:0;331c7316141f:41605 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-08T02:25:35,427 INFO [M:0;331c7316141f:41605 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T02:25:35,428 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T02:25:35,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T02:25:35,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:25:35,435 DEBUG [M:0;331c7316141f:41605 {}] zookeeper.ZKUtil(347): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T02:25:35,435 WARN [M:0;331c7316141f:41605 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T02:25:35,435 INFO [M:0;331c7316141f:41605 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/.lastflushedseqids 2024-11-08T02:25:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742477_1653 (size=72) 2024-11-08T02:25:35,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T02:25:35,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34325-0x101184078e70001, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T02:25:35,515 INFO [RS:0;331c7316141f:34325 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T02:25:35,515 INFO [RS:0;331c7316141f:34325 {}] regionserver.HRegionServer(1031): Exiting; stopping=331c7316141f,34325,1731032548147; zookeeper connection closed. 2024-11-08T02:25:35,516 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a31832b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a31832b 2024-11-08T02:25:35,518 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-08T02:25:35,850 INFO [M:0;331c7316141f:41605 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T02:25:35,851 INFO [M:0;331c7316141f:41605 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T02:25:35,851 DEBUG [M:0;331c7316141f:41605 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T02:25:35,851 INFO [M:0;331c7316141f:41605 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:25:35,851 DEBUG [M:0;331c7316141f:41605 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:25:35,851 DEBUG [M:0;331c7316141f:41605 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T02:25:35,852 DEBUG [M:0;331c7316141f:41605 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:25:35,852 INFO [M:0;331c7316141f:41605 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=751.12 KB heapSize=923.09 KB 2024-11-08T02:25:35,872 DEBUG [M:0;331c7316141f:41605 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b81a38650c24463ea4b1070a6c0814ae is 82, key is hbase:meta,,1/info:regioninfo/1731032552019/Put/seqid=0 2024-11-08T02:25:35,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742478_1654 (size=5672) 2024-11-08T02:25:36,276 INFO [M:0;331c7316141f:41605 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b81a38650c24463ea4b1070a6c0814ae 2024-11-08T02:25:36,304 DEBUG [M:0;331c7316141f:41605 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bae268e503504c768163e17cc6be1ad4 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1731032708579/Put/seqid=0 2024-11-08T02:25:36,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742479_1655 (size=44546) 2024-11-08T02:25:36,714 INFO [M:0;331c7316141f:41605 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750.56 KB at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bae268e503504c768163e17cc6be1ad4 2024-11-08T02:25:36,724 INFO [M:0;331c7316141f:41605 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bae268e503504c768163e17cc6be1ad4 2024-11-08T02:25:36,740 DEBUG [M:0;331c7316141f:41605 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/480d68817d76471fb81af798e501698d is 69, key is 331c7316141f,34325,1731032548147/rs:state/1731032551163/Put/seqid=0 2024-11-08T02:25:36,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073742480_1656 (size=5156) 2024-11-08T02:25:37,145 INFO [M:0;331c7316141f:41605 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2117 (bloomFilter=true), to=hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/480d68817d76471fb81af798e501698d 2024-11-08T02:25:37,155 DEBUG [M:0;331c7316141f:41605 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b81a38650c24463ea4b1070a6c0814ae as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b81a38650c24463ea4b1070a6c0814ae 2024-11-08T02:25:37,159 INFO [M:0;331c7316141f:41605 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b81a38650c24463ea4b1070a6c0814ae, entries=8, sequenceid=2117, filesize=5.5 K 2024-11-08T02:25:37,160 DEBUG [M:0;331c7316141f:41605 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bae268e503504c768163e17cc6be1ad4 as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bae268e503504c768163e17cc6be1ad4 2024-11-08T02:25:37,163 INFO [M:0;331c7316141f:41605 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bae268e503504c768163e17cc6be1ad4 2024-11-08T02:25:37,163 INFO [M:0;331c7316141f:41605 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bae268e503504c768163e17cc6be1ad4, entries=172, sequenceid=2117, filesize=43.5 K 2024-11-08T02:25:37,164 DEBUG [M:0;331c7316141f:41605 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/480d68817d76471fb81af798e501698d as hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/480d68817d76471fb81af798e501698d 2024-11-08T02:25:37,167 INFO [M:0;331c7316141f:41605 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43585/user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/480d68817d76471fb81af798e501698d, entries=1, sequenceid=2117, filesize=5.0 K 2024-11-08T02:25:37,168 INFO [M:0;331c7316141f:41605 {}] regionserver.HRegion(3140): Finished flush of dataSize ~751.12 KB/769146, heapSize ~922.79 KB/944936, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1316ms, sequenceid=2117, compaction requested=false 2024-11-08T02:25:37,170 INFO [M:0;331c7316141f:41605 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:25:37,170 DEBUG [M:0;331c7316141f:41605 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731032735851Disabling compacts and flushes for region at 1731032735851Disabling writes for close at 1731032735852 (+1 ms)Obtaining lock to block concurrent updates at 1731032735852Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731032735852Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=769146, getHeapSize=945176, getOffHeapSize=0, getCellsCount=2314 at 1731032735853 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731032735854 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731032735854Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731032735872 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731032735872Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731032736285 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731032736303 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731032736303Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731032736724 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731032736740 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731032736740Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4200ba80: reopening flushed file at 1731032737154 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@505ab19: reopening flushed file at 1731032737159 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16f97098: reopening flushed file at 1731032737163 (+4 ms)Finished flush of dataSize ~751.12 KB/769146, heapSize ~922.79 KB/944936, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1316ms, sequenceid=2117, compaction requested=false at 1731032737168 (+5 ms)Writing region close event to WAL at 1731032737170 (+2 ms)Closed at 1731032737170 2024-11-08T02:25:37,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40173 is added to blk_1073741830_1006 (size=907958) 2024-11-08T02:25:37,173 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7ec28e3b-769a-525f-5b37-ddf69aac49ee/MasterData/WALs/331c7316141f,41605,1731032547397/331c7316141f%2C41605%2C1731032547397.1731032550231 not finished, retry = 0 2024-11-08T02:25:37,274 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T02:25:37,275 INFO [M:0;331c7316141f:41605 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T02:25:37,275 INFO [M:0;331c7316141f:41605 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41605 2024-11-08T02:25:37,275 INFO [M:0;331c7316141f:41605 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T02:25:37,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T02:25:37,398 INFO [M:0;331c7316141f:41605 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T02:25:37,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41605-0x101184078e70000, quorum=127.0.0.1:52970, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T02:25:37,404 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T02:25:37,407 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T02:25:37,407 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T02:25:37,408 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T02:25:37,408 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/hadoop.log.dir/,STOPPED} 2024-11-08T02:25:37,411 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T02:25:37,411 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T02:25:37,411 WARN [BP-952722920-172.17.0.2-1731032543671 heartbeating to localhost/127.0.0.1:43585 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T02:25:37,411 WARN [BP-952722920-172.17.0.2-1731032543671 heartbeating to localhost/127.0.0.1:43585 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-952722920-172.17.0.2-1731032543671 (Datanode Uuid c25ea9b5-d729-450a-a489-7c0aead6d2a0) service to localhost/127.0.0.1:43585 2024-11-08T02:25:37,414 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/cluster_2d5389f6-e880-a508-0df3-b950eb02e0c6/data/data1/current/BP-952722920-172.17.0.2-1731032543671 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T02:25:37,414 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/cluster_2d5389f6-e880-a508-0df3-b950eb02e0c6/data/data2/current/BP-952722920-172.17.0.2-1731032543671 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T02:25:37,415 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T02:25:37,423 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T02:25:37,423 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T02:25:37,424 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T02:25:37,424 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T02:25:37,424 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6f2bdad3-eb7e-9aa3-c7f7-37e51aa50ca4/hadoop.log.dir/,STOPPED} 2024-11-08T02:25:37,442 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T02:25:37,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T02:25:37,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T02:25:37,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T02:25:37,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T02:25:37,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees